github.com/jenkins-x/jx/v2@v2.1.155/pkg/cmd/step/step_stash.go (about) 1 package step 2 3 import ( 4 "fmt" 5 "os" 6 "path/filepath" 7 "strings" 8 9 "github.com/jenkins-x/jx/v2/pkg/builds" 10 11 "github.com/jenkins-x/jx/v2/pkg/cmd/opts/step" 12 13 "github.com/jenkins-x/jx/v2/pkg/cmd/helper" 14 "github.com/jenkins-x/jx/v2/pkg/kube/naming" 15 16 "github.com/jenkins-x/jx/v2/pkg/collector" 17 "github.com/jenkins-x/jx/v2/pkg/gits" 18 19 jenkinsv1 "github.com/jenkins-x/jx-api/pkg/apis/jenkins.io/v1" 20 "github.com/jenkins-x/jx-logging/pkg/log" 21 "github.com/jenkins-x/jx/v2/pkg/kube" 22 "github.com/jenkins-x/jx/v2/pkg/util" 23 24 "github.com/pkg/errors" 25 26 "github.com/jenkins-x/jx/v2/pkg/cmd/opts" 27 "github.com/jenkins-x/jx/v2/pkg/cmd/templates" 28 "github.com/spf13/cobra" 29 ) 30 31 // StepStashOptions contains the command line flags 32 type StepStashOptions struct { 33 step.StepOptions 34 Pattern []string 35 Dir string 36 ToPath string 37 Basedir string 38 StorageLocation jenkinsv1.StorageLocation 39 ProjectGitURL string 40 ProjectBranch string 41 } 42 43 const ( 44 envVarSourceURL = "SOURCE_URL" 45 46 // storageSupportDescription common text for long command descriptions around storage 47 StorageSupportDescription = ` 48 Currently Jenkins X supports storing files into a branch of a git repository or in cloud blob storage like S3, GCS, Azure blobs etc. 49 50 When using Cloud Storage we use URLs like 's3://nameOfBucket' on AWS, 'gs://anotherBucket' on GCP or on Azure 'azblob://thatBucket' 51 ` 52 ) 53 54 var ( 55 stepStashLong = templates.LongDesc(` 56 This pipeline step stashes the specified files from the build into some stable storage location. 57 ` + StorageSupportDescription + helper.SeeAlsoText("jx step unstash", "jx edit storage")) 58 59 stepStashExample = templates.Examples(` 60 # lets collect some files to the team's default storage location (which if not configured uses the current git repository's gh-pages branch) 61 jx step stash -c tests -p "target/test-reports/*" 62 63 # lets collect some files to a specific Git URL for a repository 64 jx step stash -c tests -p "target/test-reports/*" --git-url https://github.com/myuser/myrepo.git 65 66 # lets collect some files with the file names relative to the 'target/test-reports' folder and store in a Git URL 67 jx step stash -c tests -p "target/test-reports/*" --basedir target/test-reports --git-url https://github.com/myuser/myrepo.git 68 69 # lets collect some files to a specific AWS cloud storage bucket 70 jx step stash -c coverage -p "build/coverage/*" --bucket-url s3://my-aws-bucket 71 72 # lets collect some files to a specific cloud storage bucket 73 jx step stash -c tests -p "target/test-reports/*" --bucket-url gs://my-gcp-bucket 74 75 # lets collect some files to a specific cloud storage bucket and specify the path to store them inside 76 jx step stash -c tests -p "target/test-reports/*" --bucket-url gs://my-gcp-bucket --to-path tests/mystuff 77 78 `) 79 ) 80 81 // NewCmdStepStash creates the CLI command 82 func NewCmdStepStash(commonOpts *opts.CommonOptions) *cobra.Command { 83 options := StepStashOptions{ 84 StepOptions: step.StepOptions{ 85 CommonOptions: commonOpts, 86 }, 87 } 88 cmd := &cobra.Command{ 89 Use: "stash", 90 Short: "Stashes local files generated as part of a pipeline into long term storage", 91 Aliases: []string{"collect"}, 92 Long: stepStashLong, 93 Example: stepStashExample, 94 Run: func(cmd *cobra.Command, args []string) { 95 options.Cmd = cmd 96 options.Args = args 97 err := options.Run() 98 helper.CheckErr(err) 99 }, 100 } 101 102 addStorageLocationFlags(cmd, &options.StorageLocation) 103 104 cmd.Flags().StringArrayVarP(&options.Pattern, "pattern", "p", nil, "Specify the pattern to use to look for files") 105 cmd.Flags().StringVarP(&options.Dir, "dir", "", "", "The source directory to try detect the current git repository or branch. Defaults to using the current directory") 106 cmd.Flags().StringVarP(&options.ToPath, "to-path", "t", "", "The path within the storage to store the files. If not specified it defaults to 'jenkins-x/$category/$owner/$repoName/$branch/$buildNumber'") 107 cmd.Flags().StringVarP(&options.Basedir, "basedir", "", "", "The base directory to use to create relative output file names. e.g. if you specify '--pattern \"target/*.xml\" then you may want to supply '--basedir target' to strip the 'target/' prefix from all collected files") 108 cmd.Flags().StringVarP(&options.ProjectGitURL, "project-git-url", "", "", "The project git URL to collect for. Used to default the organisation and repository folders in the storage. If not specified its discovered from the local '.git' folder") 109 cmd.Flags().StringVarP(&options.ProjectBranch, "project-branch", "", "", "The project git branch of the project to collect for. Used to default the branch folder in the storage. If not specified its discovered from the local '.git' folder") 110 return cmd 111 } 112 113 func addStorageLocationFlags(cmd *cobra.Command, location *jenkinsv1.StorageLocation) { 114 cmd.Flags().StringVarP(&location.Classifier, "classifier", "c", "", "A name which classifies this type of file. Example values: "+kube.ClassificationValues) 115 cmd.Flags().StringVarP(&location.BucketURL, "bucket-url", "", "", "Specify the cloud storage bucket URL to send each file to. e.g. use 's3://nameOfBucket' on AWS, gs://anotherBucket' on GCP or on Azure 'azblob://thatBucket'") 116 cmd.Flags().StringVarP(&location.GitURL, "git-url", "", "", "Specify the Git URL to of the repository to use for storage") 117 cmd.Flags().StringVarP(&location.GitBranch, "git-branch", "", "gh-pages", "The branch to use to store files in the git repository") 118 } 119 120 // Run runs the command 121 func (o *StepStashOptions) Run() error { 122 if len(o.Pattern) == 0 { 123 return util.MissingOption("pattern") 124 } 125 classifier := o.StorageLocation.Classifier 126 if classifier == "" { 127 return util.MissingOption("classifier") 128 } 129 var err error 130 if o.Dir == "" { 131 o.Dir, err = os.Getwd() 132 if err != nil { 133 return err 134 } 135 } 136 settings, err := o.TeamSettings() 137 if err != nil { 138 return err 139 } 140 if o.StorageLocation.IsEmpty() { 141 // lets try get the location from the team settings 142 o.StorageLocation = settings.StorageLocationOrDefault(classifier) 143 144 if o.StorageLocation.IsEmpty() { 145 // we have no team settings so lets try detect the git repository using an env var or local file system 146 sourceURL := os.Getenv(envVarSourceURL) 147 if sourceURL == "" { 148 _, gitConf, err := o.Git().FindGitConfigDir(o.Dir) 149 if err != nil { 150 log.Logger().Warnf("Could not find a .git directory: %s", err) 151 } else { 152 sourceURL, err = o.DiscoverGitURL(gitConf) 153 } 154 } 155 if sourceURL == "" { 156 return fmt.Errorf("Missing option --git-url and we could not detect the current git repository URL") 157 } 158 o.StorageLocation.GitURL = sourceURL 159 } 160 } 161 if o.StorageLocation.IsEmpty() { 162 return fmt.Errorf("Missing option --git-url and we could not detect the current git repository URL") 163 } 164 165 var gitKind string 166 if o.StorageLocation.GitURL != "" { 167 gitInfo, err := gits.ParseGitURL(o.StorageLocation.GitURL) 168 if err != nil { 169 return errors.Wrapf(err, "could not parse git URL for storage URL %s", o.StorageLocation.GitURL) 170 } 171 gitKind, err = o.GitServerKind(gitInfo) 172 if err != nil { 173 return errors.Wrapf(err, "could not determine git kind for storage URL %s", o.StorageLocation.GitURL) 174 } 175 } 176 177 coll, err := collector.NewCollector(o.StorageLocation, o.Git(), gitKind) 178 if err != nil { 179 return errors.Wrapf(err, "failed to create the collector for storage settings %s", o.StorageLocation.Description()) 180 } 181 182 client, ns, err := o.JXClientAndDevNamespace() 183 if err != nil { 184 return errors.Wrap(err, "cannot create the JX client") 185 } 186 187 buildNo := builds.GetBuildNumber() 188 var projectGitInfo *gits.GitRepository 189 gitURL := o.ProjectGitURL 190 if gitURL == "" { 191 gitURL = o.StorageLocation.GitURL 192 } 193 if gitURL != "" { 194 projectGitInfo, err = gits.ParseGitURL(gitURL) 195 if err != nil { 196 return errors.Wrapf(err, "failed to parse the git URL %s", gitURL) 197 } 198 } else { 199 dir := "" 200 projectGitInfo, err = o.FindGitInfo(dir) 201 if err != nil { 202 return errors.Wrapf(err, "failed to find the git information in the directory %s", dir) 203 } 204 } 205 projectOrg := projectGitInfo.Organisation 206 projectRepoName := projectGitInfo.Name 207 208 projectBranchName, err := o.determineProjectBranchName(o.ProjectBranch, gitURL) 209 if err != nil { 210 return err 211 } 212 213 storagePath := o.ToPath 214 if storagePath == "" { 215 storagePath = filepath.Join("jenkins-x", classifier, projectOrg, projectRepoName, projectBranchName, buildNo) 216 } 217 218 urls, err := coll.CollectFiles(o.Pattern, storagePath, o.Basedir) 219 if err != nil { 220 return errors.Wrapf(err, "failed to collect patterns %s to path %s", strings.Join(o.Pattern, ", "), storagePath) 221 } 222 223 for _, u := range urls { 224 log.Logger().Infof("stashed: %s", util.ColorInfo(u)) 225 } 226 227 // TODO this pipeline name construction needs moving to a shared lib, and other things refactoring to use it 228 pipeline := fmt.Sprintf("%s-%s-%s-%s", projectOrg, projectRepoName, projectBranchName, buildNo) 229 230 if pipeline != "" && buildNo != "" { 231 name := naming.ToValidName(pipeline) 232 key := &kube.PromoteStepActivityKey{ 233 PipelineActivityKey: kube.PipelineActivityKey{ 234 Name: name, 235 Pipeline: pipeline, 236 Build: buildNo, 237 GitInfo: &gits.GitRepository{ 238 Organisation: projectOrg, 239 Name: projectRepoName, 240 }, 241 }, 242 } 243 a, _, err := key.GetOrCreate(client, ns) 244 if err != nil { 245 return err 246 } 247 a.Spec.Attachments = append(a.Spec.Attachments, jenkinsv1.Attachment{ 248 Name: classifier, 249 URLs: urls, 250 }) 251 _, err = client.JenkinsV1().PipelineActivities(ns).PatchUpdate(a) 252 if err != nil { 253 return err 254 } 255 } 256 return nil 257 } 258 259 func (o *StepStashOptions) determineProjectBranchName(projectBranchName string, gitURL string) (string, error) { 260 if projectBranchName != "" { 261 return projectBranchName, nil 262 } 263 // If there isn't a bucket URL, use the configured git branch 264 if o.StorageLocation.BucketURL == "" { 265 return o.StorageLocation.GitBranch, nil 266 } 267 // If there is a bucket URL, try using the BRANCH_NAME env var. 268 if projectBranchName == "" { 269 projectBranchName = os.Getenv(util.EnvVarBranchName) 270 } 271 if projectBranchName == "" { 272 // lets try find the branch name via git 273 if gitURL == "" { 274 var err error 275 projectBranchName, err = o.Git().Branch(o.Dir) 276 if err != nil { 277 return "", err 278 } 279 } 280 } 281 282 if projectBranchName == "" { 283 return "", fmt.Errorf("environment variable %s is empty, and couldn't find a branch from %s as a git repository", util.EnvVarBranchName, o.Dir) 284 } 285 286 return projectBranchName, nil 287 }