github.com/jenkins-x/jx/v2@v2.1.155/pkg/cmd/importcmd/import.go (about) 1 package importcmd 2 3 import ( 4 "fmt" 5 "io/ioutil" 6 "os" 7 "path/filepath" 8 "strings" 9 "time" 10 11 jenkinsio "github.com/jenkins-x/jx-api/pkg/apis/jenkins.io" 12 "github.com/jenkins-x/jx/v2/pkg/cmd/step/create/pr" 13 "github.com/jenkins-x/jx/v2/pkg/maven" 14 "github.com/spf13/viper" 15 16 "github.com/cenkalti/backoff" 17 "github.com/denormal/go-gitignore" 18 gojenkins "github.com/jenkins-x/golang-jenkins" 19 v1 "github.com/jenkins-x/jx-api/pkg/apis/jenkins.io/v1" 20 "github.com/jenkins-x/jx-logging/pkg/log" 21 "github.com/jenkins-x/jx/v2/pkg/auth" 22 "github.com/jenkins-x/jx/v2/pkg/cloud/amazon" 23 "github.com/jenkins-x/jx/v2/pkg/cmd/edit" 24 "github.com/jenkins-x/jx/v2/pkg/cmd/helper" 25 "github.com/jenkins-x/jx/v2/pkg/cmd/initcmd" 26 "github.com/jenkins-x/jx/v2/pkg/cmd/opts" 27 "github.com/jenkins-x/jx/v2/pkg/cmd/start" 28 "github.com/jenkins-x/jx/v2/pkg/cmd/templates" 29 "github.com/jenkins-x/jx/v2/pkg/github" 30 "github.com/jenkins-x/jx/v2/pkg/gits" 31 "github.com/jenkins-x/jx/v2/pkg/jenkins" 32 "github.com/jenkins-x/jx/v2/pkg/jenkinsfile" 33 "github.com/jenkins-x/jx/v2/pkg/kube" 34 "github.com/jenkins-x/jx/v2/pkg/kube/naming" 35 "github.com/jenkins-x/jx/v2/pkg/prow" 36 "github.com/jenkins-x/jx/v2/pkg/util" 37 "github.com/pkg/errors" 38 "github.com/spf13/cobra" 39 "gopkg.in/AlecAivazis/survey.v1" 40 gitcfg "gopkg.in/src-d/go-git.v4/config" 41 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 42 "sigs.k8s.io/yaml" 43 ) 44 45 // CallbackFn callback function 46 type CallbackFn func() error 47 48 // ImportOptions options struct for jx import 49 type ImportOptions struct { 50 *opts.CommonOptions 51 52 RepoURL string 53 54 Dir string 55 Organisation string 56 Repository string 57 Credentials string 58 AppName string 59 GitHub bool 60 DryRun bool 61 SelectAll bool 62 DisableDraft bool 63 DisableJenkinsfileCheck bool 64 DisableWebhooks bool 65 SelectFilter string 66 Jenkinsfile string 67 BranchPattern string 68 GitRepositoryOptions gits.GitRepositoryOptions 69 ImportGitCommitMessage string 70 ListDraftPacks bool 71 DraftPack string 72 DockerRegistryOrg string 73 GitDetails gits.CreateRepoData 74 DeployKind string 75 DeployOptions v1.DeployOptions 76 SchedulerName string 77 78 DisableDotGitSearch bool 79 InitialisedGit bool 80 Jenkins gojenkins.JenkinsClient 81 GitConfDir string 82 GitServer *auth.AuthServer 83 GitUserAuth *auth.UserAuth 84 GitProvider gits.GitProvider 85 PostDraftPackCallback CallbackFn 86 DisableMaven bool 87 PipelineUserName string 88 PipelineServer string 89 ImportMode string 90 UseDefaultGit bool 91 GithubAppInstalled bool 92 PreviewNamespace string 93 reporter ImportReporter 94 } 95 96 var ( 97 importLong = templates.LongDesc(` 98 Imports a local folder or Git repository into Jenkins X. 99 100 If you specify no other options or arguments then the current directory is imported. 101 Or you can use '--dir' to specify a directory to import. 102 103 You can specify the git URL as an argument. 104 105 For more documentation see: [https://jenkins-x.io/docs/using-jx/creating/import/](https://jenkins-x.io/docs/using-jx/creating/import/) 106 107 ` + helper.SeeAlsoText("jx create project")) 108 109 importExample = templates.Examples(` 110 # Import the current folder 111 jx import 112 113 # Import a different folder 114 jx import /foo/bar 115 116 # Import a Git repository from a URL 117 jx import --url https://github.com/jenkins-x/spring-boot-web-example.git 118 119 # Select a number of repositories from a GitHub organisation 120 jx import --github --org myname 121 122 # Import all repositories from a GitHub organisation selecting ones to not import 123 jx import --github --org myname --all 124 125 # Import all repositories from a GitHub organisation which contain the text foo 126 jx import --github --org myname --all --filter foo 127 `) 128 129 deployKinds = []string{opts.DeployKindKnative, opts.DeployKindDefault} 130 131 removeSourceRepositoryAnnotations = []string{"kubectl.kubernetes.io/last-applied-configuration", "jenkins.io/chart"} 132 ) 133 134 // NewCmdImport the cobra command for jx import 135 func NewCmdImport(commonOpts *opts.CommonOptions) *cobra.Command { 136 cmd, _ := NewCmdImportAndOptions(commonOpts) 137 return cmd 138 } 139 140 // NewCmdImportAndOptions creates the cobra command for jx import and the options 141 func NewCmdImportAndOptions(commonOpts *opts.CommonOptions) (*cobra.Command, *ImportOptions) { 142 options := &ImportOptions{ 143 CommonOptions: commonOpts, 144 } 145 cmd := &cobra.Command{ 146 Use: "import", 147 Short: "Imports a local project or Git repository into Jenkins", 148 Long: importLong, 149 Example: importExample, 150 Run: func(cmd *cobra.Command, args []string) { 151 options.Cmd = cmd 152 options.Args = args 153 err := options.Run() 154 helper.CheckErr(err) 155 }, 156 } 157 cmd.Flags().StringVarP(&options.RepoURL, "url", "u", "", "The git clone URL to clone into the current directory and then import") 158 cmd.Flags().BoolVarP(&options.GitHub, "github", "", false, "If you wish to pick the repositories from GitHub to import") 159 cmd.Flags().BoolVarP(&options.SelectAll, "all", "", false, "If selecting projects to import from a Git provider this defaults to selecting them all") 160 cmd.Flags().StringVarP(&options.SelectFilter, "filter", "", "", "If selecting projects to import from a Git provider this filters the list of repositories") 161 options.AddImportFlags(cmd, false) 162 options.Cmd = cmd 163 return cmd, options 164 } 165 166 func (options *ImportOptions) AddImportFlags(cmd *cobra.Command, createProject bool) { 167 notCreateProject := func(text string) string { 168 if createProject { 169 return "" 170 } 171 return text 172 } 173 cmd.Flags().StringVarP(&options.Organisation, "org", "", "", "Specify the Git provider organisation to import the project into (if it is not already in one)") 174 cmd.Flags().StringVarP(&options.Repository, "name", "", notCreateProject("n"), "Specify the Git repository name to import the project into (if it is not already in one)") 175 cmd.Flags().StringVarP(&options.Credentials, "credentials", notCreateProject("c"), "", "The Jenkins credentials name used by the job") 176 cmd.Flags().StringVarP(&options.Jenkinsfile, "jenkinsfile", notCreateProject("j"), "", "The name of the Jenkinsfile to use. If not specified then 'Jenkinsfile' will be used") 177 cmd.Flags().BoolVarP(&options.DryRun, "dry-run", "", false, "Performs local changes to the repo but skips the import into Jenkins X") 178 cmd.Flags().BoolVarP(&options.DisableDraft, "no-draft", "", false, "Disable Draft from trying to default a Dockerfile and Helm Chart") 179 cmd.Flags().BoolVarP(&options.DisableJenkinsfileCheck, "no-jenkinsfile", "", false, "Disable defaulting a Jenkinsfile if its missing") 180 cmd.Flags().StringVarP(&options.ImportGitCommitMessage, "import-commit-message", "", "", "Specifies the initial commit message used when importing the project") 181 cmd.Flags().StringVarP(&options.BranchPattern, "branches", "", "", "The branch pattern for branches to trigger CI/CD pipelines on") 182 cmd.Flags().BoolVarP(&options.ListDraftPacks, "list-packs", "", false, "list available draft packs") 183 cmd.Flags().StringVarP(&options.DraftPack, "pack", "", "", "The name of the pack to use") 184 cmd.Flags().StringVarP(&options.SchedulerName, "scheduler", "", "", "The name of the Scheduler configuration to use for ChatOps when using Prow") 185 cmd.Flags().StringVarP(&options.DockerRegistryOrg, "docker-registry-org", "", "", "The name of the docker registry organisation to use. If not specified then the Git provider organisation will be used") 186 cmd.Flags().StringVarP(&options.ExternalJenkinsBaseURL, "external-jenkins-url", "", "", "The jenkins url that an external git provider needs to use") 187 cmd.Flags().BoolVarP(&options.DisableMaven, "disable-updatebot", "", false, "disable updatebot-maven-plugin from attempting to fix/update the maven pom.xml") 188 cmd.Flags().StringVarP(&options.ImportMode, "import-mode", "m", "", fmt.Sprintf("The import mode to use. Should be one of %s", strings.Join(v1.ImportModeStrings, ", "))) 189 cmd.Flags().BoolVarP(&options.UseDefaultGit, "use-default-git", "", false, "use default git account") 190 cmd.Flags().StringVarP(&options.DeployKind, "deploy-kind", "", "", fmt.Sprintf("The kind of deployment to use for the project. Should be one of %s", strings.Join(deployKinds, ", "))) 191 cmd.Flags().BoolVarP(&options.DeployOptions.Canary, opts.OptionCanary, "", false, "should we use canary rollouts (progressive delivery) by default for this application. e.g. using a Canary deployment via flagger. Requires the installation of flagger and istio/gloo in your cluster") 192 cmd.Flags().BoolVarP(&options.DeployOptions.HPA, opts.OptionHPA, "", false, "should we enable the Horizontal Pod Autoscaler for this application.") 193 cmd.Flags().StringVarP(&options.PreviewNamespace, "preview-namespace", "", "", "The namespace to deploy application previews into") 194 195 opts.AddGitRepoOptionsArguments(cmd, &options.GitRepositoryOptions) 196 } 197 198 // Run executes the command 199 func (options *ImportOptions) Run() error { 200 if options.ListDraftPacks { 201 packs, err := options.allDraftPacks() 202 if err != nil { 203 log.Logger().Error(err.Error()) 204 return err 205 } 206 log.Logger().Info("Available draft packs:") 207 for i := 0; i < len(packs); i++ { 208 log.Logger().Infof(packs[i]) 209 } 210 return nil 211 } 212 213 options.SetBatchMode(options.BatchMode) 214 215 var err error 216 isProw := false 217 jxClient, ns, err := options.JXClientAndDevNamespace() 218 if err != nil { 219 return err 220 } 221 if !options.DryRun { 222 _, err = options.KubeClient() 223 if err != nil { 224 return err 225 } 226 227 isProw, err = options.IsProw() 228 if err != nil { 229 return err 230 } 231 232 if !isProw { 233 options.Jenkins, err = options.JenkinsClient() 234 if err != nil { 235 return err 236 } 237 } 238 } 239 err = options.DefaultsFromTeamSettings() 240 if err != nil { 241 return err 242 } 243 244 var userAuth *auth.UserAuth 245 if options.GitProvider == nil { 246 authConfigSvc, err := options.GitLocalAuthConfigService() 247 if err != nil { 248 return err 249 } 250 config := authConfigSvc.Config() 251 var server *auth.AuthServer 252 if options.RepoURL != "" { 253 gitInfo, err := gits.ParseGitURL(options.RepoURL) 254 if err != nil { 255 return err 256 } 257 serverURL := gitInfo.HostURLWithoutUser() 258 server = config.GetOrCreateServer(serverURL) 259 } else { 260 server, err = config.PickOrCreateServer(gits.GitHubURL, options.GitRepositoryOptions.ServerURL, "Which Git service do you wish to use", options.BatchMode, options.GetIOFileHandles()) 261 if err != nil { 262 return err 263 } 264 } 265 266 if options.UseDefaultGit { 267 userAuth = config.CurrentUser(server, options.CommonOptions.InCluster()) 268 } else if options.GitRepositoryOptions.Username != "" { 269 userAuth = config.GetOrCreateUserAuth(server.URL, options.GitRepositoryOptions.Username) 270 options.GetReporter().UsingGitUserName(options.GitRepositoryOptions.Username) 271 } else { 272 // Get the org in case there is more than one user auth on the server and batchMode is true 273 org := options.getOrganisationOrCurrentUser() 274 userAuth, err = config.PickServerUserAuth(server, "Git user name:", options.BatchMode, org, options.GetIOFileHandles()) 275 if err != nil { 276 return err 277 } 278 } 279 if server.Kind == "" { 280 server.Kind, err = options.GitServerHostURLKind(server.URL) 281 if err != nil { 282 return err 283 } 284 } 285 if userAuth.IsInvalid() { 286 f := func(username string) error { 287 options.Git().PrintCreateRepositoryGenerateAccessToken(server, username, options.Out) 288 return nil 289 } 290 if options.GitRepositoryOptions.ApiToken != "" { 291 userAuth.ApiToken = options.GitRepositoryOptions.ApiToken 292 } 293 err = config.EditUserAuth(server.Label(), userAuth, userAuth.Username, false, options.BatchMode, f, options.GetIOFileHandles()) 294 if err != nil { 295 return err 296 } 297 298 // TODO lets verify the auth works? 299 if userAuth.IsInvalid() { 300 return fmt.Errorf("Authentication has failed for user %v. Please check the user's access credentials and try again", userAuth.Username) 301 } 302 } 303 err = authConfigSvc.SaveUserAuth(server.URL, userAuth) 304 if err != nil { 305 return fmt.Errorf("Failed to store git auth configuration %s", err) 306 } 307 308 options.GitServer = server 309 options.GitUserAuth = userAuth 310 options.GitProvider, err = gits.CreateProvider(server, userAuth, options.Git()) 311 if err != nil { 312 return err 313 } 314 } 315 316 if options.GitHub { 317 return options.ImportProjectsFromGitHub() 318 } 319 320 if options.Dir == "" { 321 args := options.Args 322 if len(args) > 0 { 323 options.Dir = args[0] 324 } else { 325 dir, err := os.Getwd() 326 if err != nil { 327 return err 328 } 329 options.Dir = dir 330 } 331 } 332 333 checkForJenkinsfile := options.Jenkinsfile == "" && !options.DisableJenkinsfileCheck 334 shouldClone := checkForJenkinsfile || !options.DisableDraft 335 336 if options.RepoURL != "" { 337 if shouldClone { 338 // Use the git user auth to clone the repo (needed for private repos etc) 339 if options.GitUserAuth == nil { 340 userAuth := options.GitProvider.UserAuth() 341 options.GitUserAuth = &userAuth 342 } 343 options.RepoURL, err = options.Git().CreateAuthenticatedURL(options.RepoURL, options.GitUserAuth) 344 if err != nil { 345 return err 346 } 347 err = options.CloneRepository() 348 if err != nil { 349 return err 350 } 351 } 352 } else { 353 err = options.DiscoverGit() 354 if err != nil { 355 return err 356 } 357 358 if options.RepoURL == "" { 359 err = options.DiscoverRemoteGitURL() 360 if err != nil { 361 return err 362 } 363 } 364 } 365 366 if options.AppName == "" { 367 if options.RepoURL != "" { 368 info, err := gits.ParseGitURL(options.RepoURL) 369 if err != nil { 370 log.Logger().Warnf("Failed to parse git URL %s : %s", options.RepoURL, err) 371 } else { 372 options.Organisation = info.Organisation 373 options.AppName = info.Name 374 } 375 } 376 } 377 if options.AppName == "" { 378 dir, err := filepath.Abs(options.Dir) 379 if err != nil { 380 return err 381 } 382 _, options.AppName = filepath.Split(dir) 383 } 384 options.AppName = naming.ToValidName(strings.ToLower(options.AppName)) 385 386 if !options.DisableDraft { 387 err = options.DraftCreate() 388 if err != nil { 389 return err 390 } 391 } 392 err = options.fixDockerIgnoreFile() 393 if err != nil { 394 return err 395 } 396 397 err = options.fixMaven() 398 if err != nil { 399 return err 400 } 401 402 err = options.setPreviewNamespace() 403 if err != nil { 404 return err 405 } 406 407 if options.RepoURL == "" { 408 if !options.DryRun { 409 err = options.CreateNewRemoteRepository() 410 if err != nil { 411 if !options.DisableDraft { 412 log.Logger().Warn("Remote repository creation failed. In order to retry consider adding '--no-draft' option.") 413 } 414 return err 415 } 416 } 417 } else { 418 if shouldClone { 419 err = options.Git().Push(options.Dir, "origin", false, "HEAD") 420 if err != nil { 421 return err 422 } 423 options.GetReporter().PushedGitRepository(options.RepoURL) 424 } 425 426 err = options.AddBotAsCollaborator() 427 if err != nil { 428 return err 429 } 430 } 431 432 if options.DryRun { 433 log.Logger().Info("dry-run so skipping import to Jenkins X") 434 return nil 435 } 436 437 if !isProw { 438 err = options.checkChartmuseumCredentialExists() 439 if err != nil { 440 return err 441 } 442 } 443 444 _, err = kube.GetOrCreateSourceRepository(jxClient, ns, options.AppName, options.Organisation, gits.SourceRepositoryProviderURL(options.GitProvider)) 445 if err != nil { 446 return errors.Wrapf(err, "creating application resource for %s", util.ColorInfo(options.AppName)) 447 } 448 449 if !options.GithubAppInstalled { 450 githubAppMode, err := options.IsGitHubAppMode() 451 if err != nil { 452 return err 453 } 454 455 if githubAppMode { 456 githubApp := &github.GithubApp{ 457 Factory: options.GetFactory(), 458 } 459 460 installed, err := githubApp.Install(options.Organisation, options.Repository, options.GetIOFileHandles(), false) 461 if err != nil { 462 return err 463 } 464 options.GithubAppInstalled = installed 465 } 466 } 467 468 return options.doImport() 469 } 470 471 // ImportProjectsFromGitHub import projects from github 472 func (options *ImportOptions) ImportProjectsFromGitHub() error { 473 repos, err := gits.PickRepositories(options.GitProvider, options.Organisation, "Which repositories do you want to import", options.SelectAll, options.SelectFilter, options.GetIOFileHandles()) 474 if err != nil { 475 return err 476 } 477 478 log.Logger().Info("Selected repositories") 479 for _, r := range repos { 480 o2 := ImportOptions{ 481 CommonOptions: options.CommonOptions, 482 Dir: options.Dir, 483 RepoURL: r.CloneURL, 484 Organisation: options.Organisation, 485 Repository: r.Name, 486 Jenkins: options.Jenkins, 487 GitProvider: options.GitProvider, 488 DisableJenkinsfileCheck: options.DisableJenkinsfileCheck, 489 DisableDraft: options.DisableDraft, 490 } 491 log.Logger().Infof("Importing repository %s", util.ColorInfo(r.Name)) 492 err = o2.Run() 493 if err != nil { 494 return err 495 } 496 } 497 return nil 498 } 499 500 // GetReporter returns the reporter interface 501 func (options *ImportOptions) GetReporter() ImportReporter { 502 if options.reporter == nil { 503 options.reporter = &LogImportReporter{} 504 } 505 return options.reporter 506 } 507 508 // SetReporter overrides the reporter interface 509 func (options *ImportOptions) SetReporter(reporter ImportReporter) { 510 options.reporter = reporter 511 } 512 513 // DraftCreate creates a draft 514 func (options *ImportOptions) DraftCreate() error { 515 // TODO this is a workaround of this draft issue: 516 // https://github.com/Azure/draft/issues/476 517 dir := options.Dir 518 var err error 519 520 defaultJenkinsfileName := jenkinsfile.Name 521 jenkinsfile := defaultJenkinsfileName 522 withRename := false 523 if options.Jenkinsfile != "" && options.Jenkinsfile != defaultJenkinsfileName { 524 jenkinsfile = options.Jenkinsfile 525 withRename = true 526 } 527 defaultJenkinsfile := filepath.Join(dir, defaultJenkinsfileName) 528 if !filepath.IsAbs(jenkinsfile) { 529 jenkinsfile = filepath.Join(dir, jenkinsfile) 530 } 531 args := &opts.InvokeDraftPack{ 532 Dir: dir, 533 CustomDraftPack: options.DraftPack, 534 Jenkinsfile: jenkinsfile, 535 DefaultJenkinsfile: defaultJenkinsfile, 536 WithRename: withRename, 537 InitialisedGit: options.InitialisedGit, 538 DisableJenkinsfileCheck: options.DisableJenkinsfileCheck, 539 } 540 options.DraftPack, err = options.InvokeDraftPack(args) 541 if err != nil { 542 return err 543 } 544 545 // lets rename the chart to be the same as our app name 546 err = options.renameChartToMatchAppName() 547 if err != nil { 548 return err 549 } 550 551 err = options.modifyDeployKind() 552 err = options.modifyDeployKind() 553 if err != nil { 554 return err 555 } 556 557 if options.PostDraftPackCallback != nil { 558 err = options.PostDraftPackCallback() 559 if err != nil { 560 return err 561 } 562 } 563 564 gitServerName, err := gits.GetHost(options.GitProvider) 565 if err != nil { 566 return err 567 } 568 569 if options.GitUserAuth == nil { 570 userAuth := options.GitProvider.UserAuth() 571 options.GitUserAuth = &userAuth 572 } 573 574 options.Organisation = options.GetOrganisation() 575 if options.Organisation == "" { 576 gitUsername := options.GitUserAuth.Username 577 options.Organisation, err = gits.GetOwner(options.BatchMode, options.GitProvider, gitUsername, options.GetIOFileHandles()) 578 if err != nil { 579 return err 580 } 581 } 582 583 if options.AppName == "" { 584 dir := options.Dir 585 _, defaultRepoName := filepath.Split(dir) 586 587 options.AppName, err = gits.GetRepoName(options.BatchMode, false, options.GitProvider, defaultRepoName, options.Organisation, options.GetIOFileHandles()) 588 if err != nil { 589 return err 590 } 591 } 592 593 dockerRegistryOrg := options.getDockerRegistryOrg() 594 err = options.ReplacePlaceholders(gitServerName, dockerRegistryOrg) 595 if err != nil { 596 return err 597 } 598 599 // Create Prow owners file 600 err = options.CreateProwOwnersFile() 601 if err != nil { 602 return err 603 } 604 err = options.CreateProwOwnersAliasesFile() 605 if err != nil { 606 return err 607 } 608 609 err = options.Git().Add(dir, "*") 610 if err != nil { 611 return err 612 } 613 err = options.Git().CommitIfChanges(dir, "Draft create") 614 if err != nil { 615 return err 616 } 617 618 options.GetReporter().DraftCreated(options.DraftPack) 619 return nil 620 } 621 622 func (options *ImportOptions) getDockerRegistryOrg() string { 623 dockerRegistryOrg := options.DockerRegistryOrg 624 if dockerRegistryOrg == "" { 625 dockerRegistryOrg = options.getOrganisationOrCurrentUser() 626 } 627 return strings.ToLower(dockerRegistryOrg) 628 } 629 630 func (options *ImportOptions) getOrganisationOrCurrentUser() string { 631 org := options.GetOrganisation() 632 if org == "" { 633 org = options.getCurrentUser() 634 } 635 return org 636 } 637 638 func (options *ImportOptions) getCurrentUser() string { 639 //walk through every file in the given dir and update the placeholders 640 var currentUser string 641 if options.GitServer != nil { 642 currentUser = options.GitServer.CurrentUser 643 if currentUser == "" { 644 if options.GitProvider != nil { 645 currentUser = options.GitProvider.CurrentUsername() 646 } 647 } 648 } 649 if currentUser == "" { 650 log.Logger().Warn("No username defined for the current Git server!") 651 currentUser = options.GitRepositoryOptions.Username 652 } 653 return currentUser 654 } 655 656 // GetOrganisation gets the organisation from the RepoURL (if in the github format of github.com/org/repo). It will 657 // do this in preference to the Organisation field (if set). If the repo URL does not implicitly specify an organisation 658 // then the Organisation specified in the options is used. 659 func (options *ImportOptions) GetOrganisation() string { 660 org := "" 661 gitInfo, err := gits.ParseGitURL(options.RepoURL) 662 if err == nil && gitInfo.Organisation != "" { 663 org = gitInfo.Organisation 664 if options.Organisation != "" && org != options.Organisation { 665 log.Logger().Warnf("organisation %s detected from URL %s. '--org %s' will be ignored", org, options.RepoURL, options.Organisation) 666 } 667 } else { 668 org = options.Organisation 669 } 670 return org 671 } 672 673 // CreateNewRemoteRepository creates a new remote repository 674 func (options *ImportOptions) CreateNewRemoteRepository() error { 675 authConfigSvc, err := options.GitLocalAuthConfigService() 676 if err != nil { 677 return err 678 } 679 680 dir := options.Dir 681 _, defaultRepoName := filepath.Split(dir) 682 683 options.GitRepositoryOptions.Owner = options.GetOrganisation() 684 details := &options.GitDetails 685 if details.RepoName == "" { 686 details, err = gits.PickNewGitRepository(options.BatchMode, authConfigSvc, defaultRepoName, &options.GitRepositoryOptions, 687 options.GitServer, options.GitUserAuth, options.Git(), options.GetIOFileHandles()) 688 if err != nil { 689 return err 690 } 691 } 692 693 repo, err := details.CreateRepository() 694 if err != nil { 695 return err 696 } 697 options.GitProvider = details.GitProvider 698 699 options.RepoURL = repo.CloneURL 700 pushGitURL, err := options.Git().CreateAuthenticatedURL(repo.CloneURL, details.User) 701 if err != nil { 702 return err 703 } 704 err = options.Git().AddRemote(dir, "origin", pushGitURL) 705 if err != nil { 706 return err 707 } 708 err = options.Git().PushMaster(dir) 709 if err != nil { 710 return err 711 } 712 repoURL := repo.HTMLURL 713 options.GetReporter().PushedGitRepository(repoURL) 714 715 return options.AddBotAsCollaborator() 716 } 717 718 // AddBotAsCollaborator adds the pipeline bot as collaborator to the repository 719 func (options *ImportOptions) AddBotAsCollaborator() error { 720 githubAppMode, err := options.IsGitHubAppMode() 721 if err != nil { 722 return err 723 } 724 725 if !githubAppMode { 726 // If the user creating the repo is not the pipeline user, add the pipeline user as a contributor to the repo 727 if options.PipelineUserName != options.GitUserAuth.Username && options.GitProvider.ServerURL() == options.PipelineServer { 728 // Make the invitation 729 err := options.GitProvider.AddCollaborator(options.PipelineUserName, options.Organisation, options.AppName) 730 if err != nil { 731 return err 732 } 733 734 authConfigSvc, err := options.GitAuthConfigService() 735 if err != nil { 736 return err 737 } 738 739 authConfig := authConfigSvc.Config() 740 pipelineServerAuth, pipelineUserAuth := authConfig.GetPipelineAuth() 741 pipelineUserProvider, err := gits.CreateProvider(pipelineServerAuth, pipelineUserAuth, options.Git()) 742 if err != nil { 743 return err 744 } 745 746 // Get all invitations for the pipeline user 747 // Wrapped in retry to not immediately fail the quickstart creation if APIs are flaky. 748 f := func() error { 749 invites, _, err := pipelineUserProvider.ListInvitations() 750 if err != nil { 751 return err 752 } 753 for _, x := range invites { 754 // Accept all invitations for the pipeline user 755 _, err = pipelineUserProvider.AcceptInvitation(*x.ID) 756 if err != nil { 757 return err 758 } 759 } 760 return nil 761 } 762 exponentialBackOff := backoff.NewExponentialBackOff() 763 timeout := 20 * time.Second 764 exponentialBackOff.MaxElapsedTime = timeout 765 exponentialBackOff.Reset() 766 err = backoff.Retry(f, exponentialBackOff) 767 if err != nil { 768 return err 769 } 770 } 771 } 772 return nil 773 } 774 775 // CloneRepository clones a repository 776 func (options *ImportOptions) CloneRepository() error { 777 url := options.RepoURL 778 if url == "" { 779 return fmt.Errorf("no Git repository URL defined") 780 } 781 gitInfo, err := gits.ParseGitURL(url) 782 if err != nil { 783 return fmt.Errorf("failed to parse Git URL %s due to: %s", url, err) 784 } 785 if gitInfo.Host == gits.GitHubHost && strings.HasPrefix(gitInfo.Scheme, "http") { 786 if !strings.HasSuffix(url, ".git") { 787 url += ".git" 788 } 789 options.RepoURL = url 790 } 791 cloneDir, err := util.CreateUniqueDirectory(options.Dir, gitInfo.Name, util.MaximumNewDirectoryAttempts) 792 if err != nil { 793 return errors.Wrapf(err, "failed to create unique directory for '%s'", options.Dir) 794 } 795 err = options.Git().Clone(url, cloneDir) 796 if err != nil { 797 return errors.Wrapf(err, "failed to clone in directory '%s'", cloneDir) 798 } 799 options.GetReporter().ClonedGitRepository(url) 800 options.Dir = cloneDir 801 return nil 802 } 803 804 // DiscoverGit checks if there is a git clone or prompts the user to import it 805 func (options *ImportOptions) DiscoverGit() error { 806 surveyOpts := survey.WithStdio(options.In, options.Out, options.Err) 807 if !options.DisableDotGitSearch { 808 root, gitConf, err := options.Git().FindGitConfigDir(options.Dir) 809 if err != nil { 810 return err 811 } 812 if root != "" { 813 if root != options.Dir { 814 options.GetReporter().Trace("Importing from directory %s as we found a .git folder there", root) 815 } 816 options.Dir = root 817 options.GitConfDir = gitConf 818 return nil 819 } 820 } 821 822 dir := options.Dir 823 if dir == "" { 824 return fmt.Errorf("no directory specified") 825 } 826 827 // lets prompt the user to initialise the Git repository 828 if !options.BatchMode { 829 options.GetReporter().Trace("The directory %s is not yet using git", util.ColorInfo(dir)) 830 flag := false 831 prompt := &survey.Confirm{ 832 Message: "Would you like to initialise git now?", 833 Default: true, 834 } 835 err := survey.AskOne(prompt, &flag, nil, surveyOpts) 836 if err != nil { 837 return err 838 } 839 if !flag { 840 return fmt.Errorf("please initialise git yourself then try again") 841 } 842 } 843 options.InitialisedGit = true 844 err := options.Git().Init(dir) 845 if err != nil { 846 return err 847 } 848 options.GitConfDir = filepath.Join(dir, ".git", "config") 849 err = options.DefaultGitIgnore() 850 if err != nil { 851 return err 852 } 853 err = options.Git().Add(dir, ".gitignore") 854 if err != nil { 855 return err 856 } 857 err = options.Git().Add(dir, "*") 858 if err != nil { 859 return err 860 } 861 862 err = options.Git().Status(dir) 863 if err != nil { 864 return err 865 } 866 867 message := options.ImportGitCommitMessage 868 if message == "" { 869 if options.BatchMode { 870 message = "Initial import" 871 } else { 872 messagePrompt := &survey.Input{ 873 Message: "Commit message: ", 874 Default: "Initial import", 875 } 876 err = survey.AskOne(messagePrompt, &message, nil, surveyOpts) 877 if err != nil { 878 return err 879 } 880 } 881 } 882 err = options.Git().CommitIfChanges(dir, message) 883 if err != nil { 884 return err 885 } 886 options.GetReporter().GitRepositoryCreated() 887 return nil 888 } 889 890 // DefaultGitIgnore creates a default .gitignore 891 func (options *ImportOptions) DefaultGitIgnore() error { 892 name := filepath.Join(options.Dir, ".gitignore") 893 exists, err := util.FileExists(name) 894 if err != nil { 895 return err 896 } 897 if !exists { 898 data := []byte(opts.DefaultGitIgnoreFile) 899 err = ioutil.WriteFile(name, data, util.DefaultWritePermissions) 900 if err != nil { 901 return fmt.Errorf("failed to write %s due to %s", name, err) 902 } 903 } 904 return nil 905 } 906 907 // DiscoverRemoteGitURL finds the git url by looking in the directory 908 // and looking for a .git/config file 909 func (options *ImportOptions) DiscoverRemoteGitURL() error { 910 gitConf := options.GitConfDir 911 if gitConf == "" { 912 return fmt.Errorf("no GitConfDir defined") 913 } 914 cfg := gitcfg.NewConfig() 915 data, err := ioutil.ReadFile(gitConf) 916 if err != nil { 917 return fmt.Errorf("failed to load %s due to %s", gitConf, err) 918 } 919 920 err = cfg.Unmarshal(data) 921 if err != nil { 922 return fmt.Errorf("failed to unmarshal %s due to %s", gitConf, err) 923 } 924 remotes := cfg.Remotes 925 if len(remotes) == 0 { 926 return nil 927 } 928 url := options.Git().GetRemoteUrl(cfg, "origin") 929 if url == "" { 930 url = options.Git().GetRemoteUrl(cfg, "upstream") 931 if url == "" { 932 url, err = options.PickGitRemoteURL(cfg) 933 if err != nil { 934 return err 935 } 936 } 937 } 938 if url != "" { 939 options.RepoURL = url 940 } 941 return nil 942 } 943 944 func (options *ImportOptions) doImport() error { 945 gitURL := options.RepoURL 946 gitProvider := options.GitProvider 947 if gitProvider == nil { 948 p, err := options.GitProviderForURL(gitURL, "user name to register webhook") 949 if err != nil { 950 return err 951 } 952 gitProvider = p 953 } 954 955 authConfigSvc, err := options.GitLocalAuthConfigService() 956 if err != nil { 957 return err 958 } 959 defaultJenkinsfileName := jenkinsfile.Name 960 jenkinsfile := options.Jenkinsfile 961 if jenkinsfile == "" { 962 jenkinsfile = defaultJenkinsfileName 963 } 964 965 dockerfileLocation := "" 966 if options.Dir != "" { 967 dockerfileLocation = filepath.Join(options.Dir, "Dockerfile") 968 } else { 969 dockerfileLocation = "Dockerfile" 970 } 971 dockerfileExists, err := util.FileExists(dockerfileLocation) 972 if err != nil { 973 return err 974 } 975 976 if dockerfileExists { 977 err = options.ensureDockerRepositoryExists() 978 if err != nil { 979 return err 980 } 981 } 982 983 isProw, err := options.IsProw() 984 if err != nil { 985 return err 986 } 987 988 githubAppMode, err := options.IsGitHubAppMode() 989 if err != nil { 990 return err 991 } 992 993 if isProw { 994 if !options.DisableWebhooks && !githubAppMode { 995 // register the webhook 996 err = options.CreateWebhookProw(gitURL, gitProvider) 997 if err != nil { 998 return err 999 } 1000 } 1001 return options.addProwConfig(gitURL, gitProvider.Kind()) 1002 } 1003 1004 return options.ImportProject(gitURL, options.Dir, jenkinsfile, options.BranchPattern, options.Credentials, false, gitProvider, authConfigSvc, false, options.BatchMode) 1005 } 1006 1007 func (options *ImportOptions) addProwConfig(gitURL string, gitKind string) error { 1008 gitInfo, err := gits.ParseGitURL(gitURL) 1009 if err != nil { 1010 return err 1011 } 1012 repo := gitInfo.Organisation + "/" + gitInfo.Name 1013 client, err := options.KubeClient() 1014 if err != nil { 1015 return err 1016 } 1017 devEnv, settings, err := options.DevEnvAndTeamSettings() 1018 if err != nil { 1019 return err 1020 } 1021 _, currentNamespace, err := options.KubeClientAndNamespace() 1022 if err != nil { 1023 return err 1024 } 1025 1026 gha, err := options.IsGitHubAppMode() 1027 if err != nil { 1028 return err 1029 } 1030 1031 if settings.IsSchedulerMode() { 1032 jxClient, _, err := options.JXClient() 1033 if err != nil { 1034 return err 1035 } 1036 callback := func(sr *v1.SourceRepository) { 1037 u := gitInfo.URLWithoutUser() 1038 sr.Spec.ProviderKind = gitKind 1039 sr.Spec.URL = u 1040 if sr.Spec.URL == "" { 1041 sr.Spec.URL = gitInfo.HttpsURL() 1042 } 1043 if sr.Spec.HTTPCloneURL == "" { 1044 sr.Spec.HTTPCloneURL = gits.HttpCloneURL(gitInfo, gitKind) 1045 } 1046 if sr.Spec.SSHCloneURL == "" { 1047 sr.Spec.SSHCloneURL = gitInfo.SSHURL 1048 } 1049 } 1050 sr, err := kube.GetOrCreateSourceRepositoryCallback(jxClient, currentNamespace, gitInfo.Name, gitInfo.Organisation, gitInfo.HostURLWithoutUser(), callback) 1051 log.Logger().Debugf("have SourceRepository: %s\n", sr.Name) 1052 1053 // lets update the Scheduler if one is specified and its different to the default 1054 schedulerName := options.SchedulerName 1055 if schedulerName != "" && schedulerName != sr.Spec.Scheduler.Name { 1056 sr.Spec.Scheduler.Name = schedulerName 1057 _, err = jxClient.JenkinsV1().SourceRepositories(currentNamespace).Update(sr) 1058 if err != nil { 1059 log.Logger().Warnf("failed to update the SourceRepository %s to add the Scheduler name %s due to: %s\n", sr.Name, schedulerName, err.Error()) 1060 } 1061 } 1062 1063 sourceGitURL, err := kube.GetRepositoryGitURL(sr) 1064 if err != nil { 1065 return errors.Wrapf(err, "failed to get the git URL for SourceRepository %s", sr.Name) 1066 } 1067 1068 devGitURL := devEnv.Spec.Source.URL 1069 if devGitURL != "" && !gha { 1070 // lets generate a PR 1071 base := devEnv.Spec.Source.Ref 1072 if base == "" { 1073 base = "master" 1074 } 1075 pro := &pr.StepCreatePrOptions{ 1076 SrcGitURL: sourceGitURL, 1077 GitURLs: []string{devGitURL}, 1078 Base: base, 1079 Fork: true, 1080 BranchName: sr.Name, 1081 } 1082 pro.CommonOptions = options.CommonOptions 1083 1084 changeFn := func(dir string, gitInfo *gits.GitRepository) ([]string, error) { 1085 return nil, writeSourceRepoToYaml(dir, sr) 1086 } 1087 1088 err := pro.CreatePullRequest("resource", changeFn) 1089 if err != nil { 1090 return errors.Wrapf(err, "failed to create Pull Request on the development environment git repository %s", devGitURL) 1091 } 1092 prURL := "" 1093 if pro.Results != nil && pro.Results.PullRequest != nil { 1094 prURL = pro.Results.PullRequest.URL 1095 } 1096 options.GetReporter().CreatedDevRepoPullRequest(prURL, devGitURL) 1097 } 1098 1099 err = options.GenerateProwConfig(currentNamespace, devEnv) 1100 if err != nil { 1101 return err 1102 } 1103 } else { 1104 err = prow.AddApplication(client, []string{repo}, currentNamespace, options.DraftPack, settings) 1105 if err != nil { 1106 return err 1107 } 1108 } 1109 1110 if !gha { 1111 startBuildOptions := start.StartPipelineOptions{ 1112 CommonOptions: options.CommonOptions, 1113 } 1114 startBuildOptions.Args = []string{fmt.Sprintf("%s/%s/%s", gitInfo.Organisation, gitInfo.Name, opts.MasterBranch)} 1115 err = startBuildOptions.Run() 1116 if err != nil { 1117 return fmt.Errorf("failed to start pipeline build: %s", err) 1118 } 1119 } 1120 1121 options.LogImportedProject(false, gitInfo) 1122 1123 return nil 1124 } 1125 1126 // writeSourceRepoToYaml marshals a SourceRepository to the given directory, making sure it can be loaded by boot. 1127 func writeSourceRepoToYaml(dir string, sr *v1.SourceRepository) error { 1128 outDir := filepath.Join(dir, "repositories", "templates") 1129 err := os.MkdirAll(outDir, util.DefaultWritePermissions) 1130 if err != nil { 1131 return errors.Wrapf(err, "failed to make directories %s", outDir) 1132 } 1133 1134 fileName := filepath.Join(outDir, sr.Name+"-sr.yaml") 1135 // lets clear the fields we don't need to save 1136 clearSourceRepositoryMetadata(&sr.ObjectMeta) 1137 // Ensure it has the type information it needs 1138 sr.APIVersion = jenkinsio.GroupAndVersion 1139 sr.Kind = "SourceRepository" 1140 1141 data, err := yaml.Marshal(&sr) 1142 if err != nil { 1143 return errors.Wrapf(err, "failed to marshal SourceRepository %s to yaml", sr.Name) 1144 } 1145 1146 err = ioutil.WriteFile(fileName, data, util.DefaultWritePermissions) 1147 if err != nil { 1148 return errors.Wrapf(err, "failed to save SourceRepository file %s", fileName) 1149 } 1150 return nil 1151 } 1152 1153 // clearSourceRepositoryMetadata clears unnecessary data 1154 func clearSourceRepositoryMetadata(meta *metav1.ObjectMeta) { 1155 meta.CreationTimestamp.Time = time.Time{} 1156 meta.Namespace = "" 1157 meta.OwnerReferences = nil 1158 meta.Finalizers = nil 1159 meta.Generation = 0 1160 meta.GenerateName = "" 1161 meta.SelfLink = "" 1162 meta.UID = "" 1163 meta.ResourceVersion = "" 1164 1165 for _, k := range removeSourceRepositoryAnnotations { 1166 delete(meta.Annotations, k) 1167 } 1168 } 1169 1170 // ensureDockerRepositoryExists for some kinds of container registry we need to pre-initialise its use such as for ECR 1171 func (options *ImportOptions) ensureDockerRepositoryExists() error { 1172 orgName := options.getOrganisationOrCurrentUser() 1173 appName := options.AppName 1174 if orgName == "" { 1175 log.Logger().Warnf("Missing organisation name!") 1176 return nil 1177 } 1178 if appName == "" { 1179 log.Logger().Warnf("Missing application name!") 1180 return nil 1181 } 1182 kubeClient, curNs, err := options.KubeClientAndNamespace() 1183 if err != nil { 1184 return err 1185 } 1186 ns, _, err := kube.GetDevNamespace(kubeClient, curNs) 1187 if err != nil { 1188 return err 1189 } 1190 1191 region, _ := kube.ReadRegion(kubeClient, ns) 1192 cm, err := kubeClient.CoreV1().ConfigMaps(ns).Get(kube.ConfigMapJenkinsDockerRegistry, metav1.GetOptions{}) 1193 if err != nil { 1194 return fmt.Errorf("Could not find ConfigMap %s in namespace %s: %s", kube.ConfigMapJenkinsDockerRegistry, ns, err) 1195 } 1196 if cm.Data != nil { 1197 dockerRegistry := cm.Data["docker.registry"] 1198 if dockerRegistry != "" { 1199 if strings.HasSuffix(dockerRegistry, ".amazonaws.com") && strings.Index(dockerRegistry, ".ecr.") > 0 { 1200 return amazon.LazyCreateRegistry(kubeClient, ns, region, dockerRegistry, options.getDockerRegistryOrg(), appName) 1201 } 1202 } 1203 } 1204 return nil 1205 } 1206 1207 // ReplacePlaceholders replaces app name, git server name, git org, and docker registry org placeholders 1208 func (options *ImportOptions) ReplacePlaceholders(gitServerName, dockerRegistryOrg string) error { 1209 options.Organisation = naming.ToValidName(strings.ToLower(options.Organisation)) 1210 options.GetReporter().Trace("replacing placeholders in directory %s", options.Dir) 1211 options.GetReporter().Trace("app name: %s, git server: %s, org: %s, Docker registry org: %s", options.AppName, gitServerName, options.Organisation, dockerRegistryOrg) 1212 1213 ignore, err := gitignore.NewRepository(options.Dir) 1214 if err != nil { 1215 return err 1216 } 1217 1218 replacer := strings.NewReplacer( 1219 util.PlaceHolderAppName, strings.ToLower(options.AppName), 1220 util.PlaceHolderGitProvider, strings.ToLower(gitServerName), 1221 util.PlaceHolderOrg, strings.ToLower(options.Organisation), 1222 util.PlaceHolderDockerRegistryOrg, strings.ToLower(dockerRegistryOrg)) 1223 1224 pathsToRename := []string{} // Renaming must be done post-Walk 1225 if err := filepath.Walk(options.Dir, func(f string, fi os.FileInfo, err error) error { 1226 if skip, err := options.skipPathForReplacement(f, fi, ignore); skip { 1227 return err 1228 } 1229 if strings.Contains(filepath.Base(f), util.PlaceHolderPrefix) { 1230 // Prepend so children are renamed before their parents 1231 pathsToRename = append([]string{f}, pathsToRename...) 1232 } 1233 if !fi.IsDir() { 1234 if err := replacePlaceholdersInFile(replacer, f); err != nil { 1235 return err 1236 } 1237 } 1238 return nil 1239 1240 }); err != nil { 1241 return fmt.Errorf("error replacing placeholders %v", err) 1242 } 1243 1244 for _, path := range pathsToRename { 1245 if err := replacePlaceholdersInPathBase(replacer, path); err != nil { 1246 return err 1247 } 1248 } 1249 return nil 1250 } 1251 1252 func (options *ImportOptions) skipPathForReplacement(path string, fi os.FileInfo, ignore gitignore.GitIgnore) (bool, error) { 1253 relPath, _ := filepath.Rel(options.Dir, path) 1254 match := ignore.Relative(relPath, fi.IsDir()) 1255 matchIgnore := match != nil && match.Ignore() //Defaults to including if match == nil 1256 if fi.IsDir() { 1257 if matchIgnore || fi.Name() == ".git" { 1258 options.GetReporter().Trace("skipping directory %q", path) 1259 return true, filepath.SkipDir 1260 } 1261 } else if matchIgnore { 1262 options.GetReporter().Trace("skipping ignored file %q", path) 1263 return true, nil 1264 } 1265 // Don't process nor follow symlinks 1266 if (fi.Mode() & os.ModeSymlink) == os.ModeSymlink { 1267 options.GetReporter().Trace("skipping symlink file %q", path) 1268 return true, nil 1269 } 1270 return false, nil 1271 } 1272 1273 func replacePlaceholdersInFile(replacer viper.StringReplacer, file string) error { 1274 input, err := ioutil.ReadFile(file) 1275 if err != nil { 1276 log.Logger().Errorf("failed to read file %s: %v", file, err) 1277 return err 1278 } 1279 1280 lines := string(input) 1281 if strings.Contains(lines, util.PlaceHolderPrefix) { // Avoid unnecessarily rewriting files 1282 output := replacer.Replace(lines) 1283 err = ioutil.WriteFile(file, []byte(output), 0600) 1284 if err != nil { 1285 log.Logger().Errorf("failed to write file %s: %v", file, err) 1286 return err 1287 } 1288 } 1289 1290 return nil 1291 } 1292 1293 func replacePlaceholdersInPathBase(replacer viper.StringReplacer, path string) error { 1294 base := filepath.Base(path) 1295 newBase := replacer.Replace(base) 1296 if newBase != base { 1297 newPath := filepath.Join(filepath.Dir(path), newBase) 1298 if err := os.Rename(path, newPath); err != nil { 1299 log.Logger().Errorf("failed to rename %q to %q: %v", path, newPath, err) 1300 return err 1301 } 1302 } 1303 return nil 1304 } 1305 1306 func (options *ImportOptions) addAppNameToGeneratedFile(filename, field, value string) error { 1307 dir := filepath.Join(options.Dir, "charts", options.AppName) 1308 file := filepath.Join(dir, filename) 1309 exists, err := util.FileExists(file) 1310 if err != nil { 1311 return err 1312 } 1313 if !exists { 1314 // no file so lets ignore this 1315 return nil 1316 } 1317 input, err := ioutil.ReadFile(file) 1318 if err != nil { 1319 return err 1320 } 1321 1322 lines := strings.Split(string(input), "\n") 1323 1324 for i, line := range lines { 1325 if strings.Contains(line, field) { 1326 lines[i] = fmt.Sprintf("%s%s", field, value) 1327 } 1328 } 1329 output := strings.Join(lines, "\n") 1330 err = ioutil.WriteFile(file, []byte(output), 0600) 1331 if err != nil { 1332 return err 1333 } 1334 return nil 1335 } 1336 1337 func (options *ImportOptions) checkChartmuseumCredentialExists() error { 1338 client, devNamespace, err := options.KubeClientAndDevNamespace() 1339 if err != nil { 1340 return err 1341 } 1342 name := jenkins.DefaultJenkinsCredentialsPrefix + jenkins.Chartmuseum 1343 secret, err := client.CoreV1().Secrets(devNamespace).Get(name, metav1.GetOptions{}) 1344 if err != nil { 1345 return fmt.Errorf("error getting %s secret %v", name, err) 1346 } 1347 1348 data := secret.Data 1349 username := string(data["BASIC_AUTH_USER"]) 1350 password := string(data["BASIC_AUTH_PASS"]) 1351 1352 if secret.Labels != nil && secret.Labels[kube.LabelCredentialsType] == kube.ValueCredentialTypeUsernamePassword { 1353 // no need to create a credential as this will be done via the kubernetes credential provider plugin 1354 return nil 1355 } 1356 1357 _, err = options.Jenkins.GetCredential(name) 1358 if err != nil { 1359 err = options.Retry(3, 10*time.Second, func() (err error) { 1360 return options.Jenkins.CreateCredential(name, username, password) 1361 }) 1362 1363 if err != nil { 1364 return fmt.Errorf("error creating Jenkins credential %s %v", name, err) 1365 } 1366 } 1367 return nil 1368 } 1369 1370 func (options *ImportOptions) renameChartToMatchAppName() error { 1371 var oldChartsDir string 1372 dir := options.Dir 1373 chartsDir := filepath.Join(dir, "charts") 1374 exists, err := util.DirExists(chartsDir) 1375 if err != nil { 1376 return errors.Wrapf(err, "failed to check if the charts directory exists %s", chartsDir) 1377 } 1378 if !exists { 1379 return nil 1380 } 1381 files, err := ioutil.ReadDir(chartsDir) 1382 if err != nil { 1383 return fmt.Errorf("error matching a Jenkins X draft pack name with chart folder %v", err) 1384 } 1385 for _, fi := range files { 1386 if fi.IsDir() { 1387 name := fi.Name() 1388 // TODO we maybe need to try check if the sub dir named after the build pack matches first? 1389 if name != "preview" && name != ".git" { 1390 oldChartsDir = filepath.Join(chartsDir, name) 1391 break 1392 } 1393 } 1394 } 1395 if oldChartsDir != "" { 1396 // chart expects folder name to be the same as app name 1397 newChartsDir := filepath.Join(dir, "charts", options.AppName) 1398 1399 exists, err := util.DirExists(oldChartsDir) 1400 if err != nil { 1401 return err 1402 } 1403 if exists && oldChartsDir != newChartsDir { 1404 err = util.RenameDir(oldChartsDir, newChartsDir, false) 1405 if err != nil { 1406 if os.IsExist(errors.Cause(err)) { 1407 isChartDir, e := util.FileExists(filepath.Join(newChartsDir, "Chart.yaml")) 1408 if e != nil { 1409 return fmt.Errorf("error checking %s chart dir: %w", options.AppName, e) 1410 } 1411 if !isChartDir { 1412 return fmt.Errorf("directory %s already exists but is not a helm chart", newChartsDir) 1413 } 1414 os.RemoveAll(oldChartsDir) 1415 log.Logger().Warnf("Failed to apply the build pack in %s: %s", dir, err) 1416 } else { 1417 return fmt.Errorf("error renaming %s to %s, %v", oldChartsDir, newChartsDir, err) 1418 } 1419 } 1420 _, err = os.Stat(newChartsDir) 1421 if err != nil { 1422 return err 1423 } 1424 } 1425 // now update the chart.yaml 1426 err = options.addAppNameToGeneratedFile("Chart.yaml", "name: ", options.AppName) 1427 if err != nil { 1428 return err 1429 } 1430 } 1431 return nil 1432 } 1433 1434 func (options *ImportOptions) fixDockerIgnoreFile() error { 1435 filename := filepath.Join(options.Dir, ".dockerignore") 1436 exists, err := util.FileExists(filename) 1437 if err == nil && exists { 1438 data, err := ioutil.ReadFile(filename) 1439 if err != nil { 1440 return fmt.Errorf("Failed to load %s: %s", filename, err) 1441 } 1442 lines := strings.Split(string(data), "\n") 1443 for i, line := range lines { 1444 if strings.TrimSpace(line) == "Dockerfile" { 1445 lines = append(lines[:i], lines[i+1:]...) 1446 text := strings.Join(lines, "\n") 1447 err = ioutil.WriteFile(filename, []byte(text), util.DefaultWritePermissions) 1448 if err != nil { 1449 return err 1450 } 1451 options.GetReporter().Trace("Removed old `Dockerfile` entry from %s", util.ColorInfo(filename)) 1452 } 1453 } 1454 } 1455 return nil 1456 } 1457 1458 // CreateProwOwnersFile creates an OWNERS file in the root of the project assigning the current Git user as an approver and a reviewer. If the file already exists, does nothing. 1459 func (options *ImportOptions) CreateProwOwnersFile() error { 1460 filename := filepath.Join(options.Dir, "OWNERS") 1461 exists, err := util.FileExists(filename) 1462 if err != nil { 1463 return err 1464 } 1465 if exists { 1466 return nil 1467 } 1468 if options.GitUserAuth != nil && options.GitUserAuth.Username != "" { 1469 data := prow.Owners{ 1470 []string{options.GitUserAuth.Username}, 1471 []string{options.GitUserAuth.Username}, 1472 } 1473 yaml, err := yaml.Marshal(&data) 1474 if err != nil { 1475 return err 1476 } 1477 err = ioutil.WriteFile(filename, yaml, 0600) 1478 if err != nil { 1479 return err 1480 } 1481 return nil 1482 } 1483 return errors.New("GitUserAuth.Username not set") 1484 } 1485 1486 // CreateProwOwnersAliasesFile creates an OWNERS_ALIASES file in the root of the project assigning the current Git user as an approver and a reviewer. 1487 func (options *ImportOptions) CreateProwOwnersAliasesFile() error { 1488 filename := filepath.Join(options.Dir, "OWNERS_ALIASES") 1489 exists, err := util.FileExists(filename) 1490 if err != nil { 1491 return err 1492 } 1493 if exists { 1494 return nil 1495 } 1496 if options.GitUserAuth == nil { 1497 return errors.New("option GitUserAuth not set") 1498 } 1499 gitUser := options.GitUserAuth.Username 1500 if gitUser != "" { 1501 data := prow.OwnersAliases{ 1502 []string{gitUser}, 1503 []string{gitUser}, 1504 []string{gitUser}, 1505 } 1506 yaml, err := yaml.Marshal(&data) 1507 if err != nil { 1508 return err 1509 } 1510 return ioutil.WriteFile(filename, yaml, 0600) 1511 } 1512 return errors.New("GitUserAuth.Username not set") 1513 } 1514 1515 func (options *ImportOptions) setPreviewNamespace() error { 1516 jxClient, ns, err := options.JXClientAndDevNamespace() 1517 if err != nil { 1518 return err 1519 } 1520 envsList, err := jxClient.JenkinsV1().Environments(ns).List(metav1.ListOptions{}) 1521 if err != nil { 1522 return err 1523 } 1524 var previewNamespaceName string 1525 if options.PreviewNamespace == "" { 1526 if options.BatchMode { 1527 return nil 1528 } 1529 surveyOpts := survey.WithStdio(options.In, options.Out, options.Err) 1530 prompt := &survey.Confirm{ 1531 Message: "Would you like to define a different preview namespace?", 1532 Default: false, 1533 } 1534 var modifyPreviewNamespace bool 1535 err := survey.AskOne(prompt, &modifyPreviewNamespace, nil, surveyOpts) 1536 if err != nil { 1537 return err 1538 } 1539 1540 if !modifyPreviewNamespace { 1541 return nil 1542 } 1543 1544 nsNamePrompt := &survey.Input{ 1545 Message: "Enter the name for the preview namespace: ", 1546 Default: "jx-previews", 1547 } 1548 1549 err = survey.AskOne(nsNamePrompt, &previewNamespaceName, func(ans interface{}) error { 1550 return isValidPreviewNamespace(ans, envsList) 1551 }, surveyOpts) 1552 if err != nil { 1553 return err 1554 } 1555 } else { 1556 if err := isValidPreviewNamespace(options.PreviewNamespace, envsList); err == nil { 1557 previewNamespaceName = options.PreviewNamespace 1558 } else { 1559 // We don't want to error at this point, we'll not define a custom namespace and ask users to modify it later in the repos 1560 log.Logger().Warnf("%s - will use the default preview namespace pattern", err.Error()) 1561 } 1562 } 1563 1564 previewsValuesFilePath := filepath.Join(options.Dir, "charts", "preview", opts.ValuesFile) 1565 exists, err := util.FileExists(previewsValuesFilePath) 1566 if err != nil { 1567 return err 1568 } 1569 if exists && previewNamespaceName != "" { 1570 log.Logger().Debugf("Modifying the default previews namespace to %s", previewNamespaceName) 1571 err = options.modifyApplicationPreviewNamespace(previewsValuesFilePath, previewNamespaceName) 1572 if err != nil { 1573 return errors.Wrap(err, "there was a problem modifying the preview chart to add a different preview namespace") 1574 } 1575 } 1576 return nil 1577 } 1578 func (options *ImportOptions) fixMaven() error { 1579 if options.DisableMaven { 1580 return nil 1581 } 1582 dir := options.Dir 1583 pomName := filepath.Join(dir, "pom.xml") 1584 exists, err := util.FileExists(pomName) 1585 if err != nil { 1586 return err 1587 } 1588 if exists { 1589 err = maven.InstallMavenIfRequired() 1590 if err != nil { 1591 return err 1592 } 1593 1594 // lets ensure the mvn plugins are ok 1595 out, err := options.GetCommandOutput(dir, "mvn", "io.jenkins.updatebot:updatebot-maven-plugin:RELEASE:plugin", "-Dartifact=maven-deploy-plugin", "-Dversion="+opts.MinimumMavenDeployVersion) 1596 if err != nil { 1597 return fmt.Errorf("Failed to update maven deploy plugin: %s output: %s", err, out) 1598 } 1599 out, err = options.GetCommandOutput(dir, "mvn", "io.jenkins.updatebot:updatebot-maven-plugin:RELEASE:plugin", "-Dartifact=maven-surefire-plugin", "-Dversion=3.0.0-M1") 1600 if err != nil { 1601 return fmt.Errorf("Failed to update maven surefire plugin: %s output: %s", err, out) 1602 } 1603 if !options.DryRun { 1604 err = options.Git().Add(dir, "pom.xml") 1605 if err != nil { 1606 return err 1607 } 1608 err = options.Git().CommitIfChanges(dir, "fix:(plugins) use a better version of maven plugins") 1609 if err != nil { 1610 return err 1611 } 1612 } 1613 1614 // lets ensure the probe paths are ok 1615 out, err = options.GetCommandOutput(dir, "mvn", "io.jenkins.updatebot:updatebot-maven-plugin:RELEASE:chart") 1616 if err != nil { 1617 return fmt.Errorf("Failed to update chart: %s output: %s", err, out) 1618 } 1619 if !options.DryRun { 1620 exists, err := util.DirExists(filepath.Join(dir, "charts")) 1621 if err != nil { 1622 return err 1623 } 1624 if exists { 1625 err = options.Git().Add(dir, "charts") 1626 if err != nil { 1627 return err 1628 } 1629 err = options.Git().CommitIfChanges(dir, "fix:(chart) fix up the probe path") 1630 if err != nil { 1631 return err 1632 } 1633 } 1634 } 1635 } 1636 return nil 1637 } 1638 1639 func (options *ImportOptions) DefaultsFromTeamSettings() error { 1640 settings, err := options.TeamSettings() 1641 if err != nil { 1642 return err 1643 } 1644 return options.DefaultValuesFromTeamSettings(settings) 1645 } 1646 1647 // DefaultValuesFromTeamSettings defaults the repository options from the given team settings 1648 func (options *ImportOptions) DefaultValuesFromTeamSettings(settings *v1.TeamSettings) error { 1649 if options.DeployKind == "" { 1650 options.DeployKind = settings.DeployKind 1651 } 1652 1653 // lets override any deploy options from the team settings if they are not specified 1654 teamDeployOptions := settings.GetDeployOptions() 1655 if !options.FlagChanged(opts.OptionCanary) { 1656 options.DeployOptions.Canary = teamDeployOptions.Canary 1657 } 1658 if !options.FlagChanged(opts.OptionHPA) { 1659 options.DeployOptions.HPA = teamDeployOptions.HPA 1660 } 1661 if options.Organisation == "" { 1662 options.Organisation = settings.Organisation 1663 } 1664 if options.GitRepositoryOptions.Owner == "" { 1665 options.GitRepositoryOptions.Owner = settings.Organisation 1666 } 1667 if options.DockerRegistryOrg == "" { 1668 options.DockerRegistryOrg = settings.DockerRegistryOrg 1669 } 1670 if options.GitRepositoryOptions.ServerURL == "" { 1671 options.GitRepositoryOptions.ServerURL = settings.GitServer 1672 } 1673 options.GitRepositoryOptions.Public = settings.GitPublic || options.GitRepositoryOptions.Public 1674 options.PipelineServer = settings.GitServer 1675 options.PipelineUserName = settings.PipelineUsername 1676 return nil 1677 } 1678 1679 func (options *ImportOptions) allDraftPacks() ([]string, error) { 1680 // lets make sure we have the latest draft packs 1681 initOpts := initcmd.InitOptions{ 1682 CommonOptions: options.CommonOptions, 1683 } 1684 log.Logger().Debug("Getting latest packs ...") 1685 dir, _, err := initOpts.InitBuildPacks(nil) 1686 if err != nil { 1687 return nil, err 1688 } 1689 1690 files, err := ioutil.ReadDir(dir) 1691 if err != nil { 1692 return nil, err 1693 } 1694 result := make([]string, 0) 1695 for _, f := range files { 1696 if f.IsDir() { 1697 result = append(result, f.Name()) 1698 } 1699 } 1700 return result, err 1701 1702 } 1703 1704 // ConfigureImportOptions updates the import options struct based on values from the create repo struct 1705 func (options *ImportOptions) ConfigureImportOptions(repoData *gits.CreateRepoData) { 1706 // configure the import options based on previous answers 1707 options.AppName = repoData.RepoName 1708 options.GitProvider = repoData.GitProvider 1709 options.Organisation = repoData.Organisation 1710 options.Repository = repoData.RepoName 1711 options.GitDetails = *repoData 1712 options.GitServer = repoData.GitServer 1713 } 1714 1715 // GetGitRepositoryDetails determines the git repository details to use during the import command 1716 func (options *ImportOptions) GetGitRepositoryDetails() (*gits.CreateRepoData, error) { 1717 err := options.DefaultsFromTeamSettings() 1718 if err != nil { 1719 return nil, err 1720 } 1721 authConfigSvc, err := options.GitLocalAuthConfigService() 1722 if err != nil { 1723 return nil, err 1724 } 1725 //config git repositoryoptions parameters: Owner and RepoName 1726 options.GitRepositoryOptions.Owner = options.Organisation 1727 options.GitRepositoryOptions.RepoName = options.Repository 1728 details, err := gits.PickNewOrExistingGitRepository(options.BatchMode, authConfigSvc, 1729 "", &options.GitRepositoryOptions, nil, nil, options.Git(), false, options.GetIOFileHandles()) 1730 if err != nil { 1731 return nil, err 1732 } 1733 return details, nil 1734 } 1735 1736 // modifyDeployKind lets modify the deployment kind if the team settings or CLI settings are different 1737 func (options *ImportOptions) modifyDeployKind() error { 1738 deployKind := options.DeployKind 1739 if deployKind == "" { 1740 return nil 1741 } 1742 dopts := options.DeployOptions 1743 1744 copy := *options.CommonOptions 1745 cmd, eo := edit.NewCmdEditDeployKindAndOption(©) 1746 eo.Dir = options.Dir 1747 1748 // lets parse the CLI arguments so that the flags are marked as specified to force them to be overridden 1749 err := cmd.Flags().Parse(edit.ToDeployArguments(opts.OptionKind, deployKind, dopts.Canary, dopts.HPA)) 1750 if err != nil { 1751 return err 1752 } 1753 err = eo.Run() 1754 if err != nil { 1755 return errors.Wrapf(err, "failed to modify the deployment kind to %s", deployKind) 1756 } 1757 return nil 1758 } 1759 1760 func isValidPreviewNamespace(ns interface{}, envs *v1.EnvironmentList) error { 1761 for _, env := range envs.Items { 1762 if ns == env.Spec.Namespace && env.Spec.Kind.IsPermanent() { 1763 return fmt.Errorf("can't use a permanent environment namespace for previews") 1764 } 1765 } 1766 return nil 1767 } 1768 1769 func (options *ImportOptions) modifyApplicationPreviewNamespace(previewsValuesFilePath string, previewNamespaceName string) error { 1770 values, err := ioutil.ReadFile(previewsValuesFilePath) 1771 if err != nil { 1772 return errors.Wrap(err, "there was a problem reading the previews values file") 1773 } 1774 var valuesMap map[string]interface{} 1775 err = yaml.Unmarshal(values, &valuesMap) 1776 if err != nil { 1777 return errors.Wrap(err, "there was a problem unmarshalling the previews values file") 1778 } 1779 if v, valExists := valuesMap["preview"]; valExists { 1780 v.(map[string]interface{})["namespace"] = previewNamespaceName 1781 } 1782 valuesB, err := yaml.Marshal(valuesMap) 1783 if err != nil { 1784 return errors.Wrap(err, "there was a problem marshalling the modified previews values file") 1785 } 1786 log.Logger().Debugf("Resulting previews values.yaml file:") 1787 log.Logger().Debugf(util.ColorStatus(string(valuesB))) 1788 1789 err = ioutil.WriteFile(previewsValuesFilePath, valuesB, util.DefaultWritePermissions) 1790 if err != nil { 1791 return errors.Wrap(err, "there was a problem writing the values file") 1792 } 1793 1794 err = options.Git().Add(options.Dir, "*") 1795 if err != nil { 1796 return err 1797 } 1798 err = options.Git().CommitIfChanges(options.Dir, "Add customized preview namespace") 1799 if err != nil { 1800 return err 1801 } 1802 return nil 1803 }