github.com/SAP/jenkins-library@v1.362.0/cmd/nexusUpload.go (about) 1 package cmd 2 3 import ( 4 "fmt" 5 piperhttp "github.com/SAP/jenkins-library/pkg/http" 6 "github.com/pkg/errors" 7 "io" 8 "net/http" 9 "os" 10 "path/filepath" 11 "strings" 12 13 b64 "encoding/base64" 14 15 "github.com/SAP/jenkins-library/pkg/command" 16 "github.com/SAP/jenkins-library/pkg/log" 17 "github.com/SAP/jenkins-library/pkg/maven" 18 "github.com/SAP/jenkins-library/pkg/nexus" 19 "github.com/SAP/jenkins-library/pkg/piperenv" 20 "github.com/SAP/jenkins-library/pkg/piperutils" 21 "github.com/SAP/jenkins-library/pkg/telemetry" 22 "github.com/ghodss/yaml" 23 ) 24 25 // nexusUploadUtils defines an interface for utility functionality used from external packages, 26 // so it can be easily mocked for testing. 27 type nexusUploadUtils interface { 28 Stdout(out io.Writer) 29 Stderr(err io.Writer) 30 SetEnv(env []string) 31 RunExecutable(e string, p ...string) error 32 33 FileExists(path string) (bool, error) 34 FileRead(path string) ([]byte, error) 35 FileWrite(path string, content []byte, perm os.FileMode) error 36 FileRemove(path string) error 37 DirExists(path string) (bool, error) 38 Glob(pattern string) (matches []string, err error) 39 Copy(src, dest string) (int64, error) 40 MkdirAll(path string, perm os.FileMode) error 41 42 DownloadFile(url, filename string, header http.Header, cookies []*http.Cookie) error 43 44 UsesMta() bool 45 UsesMaven() bool 46 UsesNpm() bool 47 48 getEnvParameter(path, name string) string 49 evaluate(options *maven.EvaluateOptions, expression string) (string, error) 50 } 51 52 type utilsBundle struct { 53 *piperutils.ProjectStructure 54 *piperutils.Files 55 *command.Command 56 *piperhttp.Client 57 } 58 59 func newUtilsBundle() *utilsBundle { 60 utils := utilsBundle{ 61 ProjectStructure: &piperutils.ProjectStructure{}, 62 Files: &piperutils.Files{}, 63 Command: &command.Command{}, 64 Client: &piperhttp.Client{}, 65 } 66 utils.Stdout(log.Writer()) 67 utils.Stderr(log.Writer()) 68 return &utils 69 } 70 71 func (u *utilsBundle) FileWrite(filePath string, content []byte, perm os.FileMode) error { 72 parent := filepath.Dir(filePath) 73 if parent != "" { 74 err := u.Files.MkdirAll(parent, 0775) 75 if err != nil { 76 return err 77 } 78 } 79 return u.Files.FileWrite(filePath, content, perm) 80 } 81 82 func (u *utilsBundle) getEnvParameter(path, name string) string { 83 return piperenv.GetParameter(path, name) 84 } 85 86 func (u *utilsBundle) evaluate(options *maven.EvaluateOptions, expression string) (string, error) { 87 return maven.Evaluate(options, expression, u) 88 } 89 90 func nexusUpload(options nexusUploadOptions, _ *telemetry.CustomData) { 91 utils := newUtilsBundle() 92 uploader := nexus.Upload{} 93 94 err := runNexusUpload(utils, &uploader, &options) 95 if err != nil { 96 log.Entry().WithError(err).Fatal("step execution failed") 97 } 98 } 99 100 func runNexusUpload(utils nexusUploadUtils, uploader nexus.Uploader, options *nexusUploadOptions) error { 101 performMavenUpload := len(options.MavenRepository) > 0 102 performNpmUpload := len(options.NpmRepository) > 0 103 104 if !performMavenUpload && !performNpmUpload { 105 if options.Format == "" { 106 return fmt.Errorf("none of the parameters 'mavenRepository' and 'npmRepository' are configured, or 'format' should be set if the 'url' already contains the repository ID") 107 } 108 if options.Format == "maven" { 109 performMavenUpload = true 110 } else if options.Format == "npm" { 111 performNpmUpload = true 112 } 113 } 114 115 err := uploader.SetRepoURL(options.Url, options.Version, options.MavenRepository, options.NpmRepository) 116 if err != nil { 117 return err 118 } 119 120 if utils.UsesNpm() && performNpmUpload { 121 log.Entry().Info("NPM project structure detected") 122 err = uploadNpmArtifacts(utils, uploader, options) 123 } else { 124 log.Entry().Info("Skipping npm upload because either no package json was found or NpmRepository option is not provided.") 125 } 126 if err != nil { 127 return err 128 } 129 130 if performMavenUpload { 131 if utils.UsesMta() { 132 log.Entry().Info("MTA project structure detected") 133 return uploadMTA(utils, uploader, options) 134 } else if utils.UsesMaven() { 135 log.Entry().Info("Maven project structure detected") 136 return uploadMaven(utils, uploader, options) 137 } 138 } else { 139 log.Entry().Info("Skipping maven and mta upload because mavenRepository option is not provided.") 140 } 141 142 return nil 143 } 144 145 func uploadNpmArtifacts(utils nexusUploadUtils, uploader nexus.Uploader, options *nexusUploadOptions) error { 146 environment := []string{"npm_config_registry=" + uploader.GetNexusURLProtocol() + "://" + uploader.GetNpmRepoURL(), "npm_config_email=project-piper@no-reply.com"} 147 if options.Username != "" && options.Password != "" { 148 auth := b64.StdEncoding.EncodeToString([]byte(options.Username + ":" + options.Password)) 149 environment = append(environment, "npm_config__auth="+auth) 150 } else { 151 log.Entry().Info("No credentials provided for npm upload, trying to upload anonymously.") 152 } 153 utils.SetEnv(environment) 154 err := utils.RunExecutable("npm", "publish") 155 return err 156 } 157 158 func uploadMTA(utils nexusUploadUtils, uploader nexus.Uploader, options *nexusUploadOptions) error { 159 if options.GroupID == "" { 160 return fmt.Errorf("the 'groupId' parameter needs to be provided for MTA projects") 161 } 162 var mtaPath string 163 exists, _ := utils.FileExists("mta.yaml") 164 if exists { 165 mtaPath = "mta.yaml" 166 // Give this file precedence, but it would be even better if 167 // ProjectStructure could be asked for the mta file it detected. 168 } else { 169 // This will fail anyway if the file doesn't exist 170 mtaPath = "mta.yml" 171 } 172 mtaInfo, err := getInfoFromMtaFile(utils, mtaPath) 173 if err == nil { 174 if options.ArtifactID != "" { 175 mtaInfo.ID = options.ArtifactID 176 } 177 err = uploader.SetInfo(options.GroupID, mtaInfo.ID, mtaInfo.Version) 178 if err == nexus.ErrEmptyVersion { 179 err = fmt.Errorf("the project descriptor file 'mta.yaml' has an invalid version: %w", err) 180 } 181 } 182 if err == nil { 183 err = addArtifact(utils, uploader, mtaPath, "", "yaml") 184 } 185 if err == nil { 186 mtarFilePath := utils.getEnvParameter(".pipeline/commonPipelineEnvironment", "mtarFilePath") 187 log.Entry().Debugf("mtar file path: '%s'", mtarFilePath) 188 err = addArtifact(utils, uploader, mtarFilePath, "", "mtar") 189 } 190 if err == nil { 191 err = uploadArtifacts(utils, uploader, options, false) 192 } 193 return err 194 } 195 196 type mtaYaml struct { 197 ID string `json:"ID"` 198 Version string `json:"version"` 199 } 200 201 func getInfoFromMtaFile(utils nexusUploadUtils, filePath string) (*mtaYaml, error) { 202 mtaYamlContent, err := utils.FileRead(filePath) 203 if err != nil { 204 return nil, fmt.Errorf("could not read from required project descriptor file '%s'", 205 filePath) 206 } 207 return getInfoFromMtaYaml(mtaYamlContent, filePath) 208 } 209 210 func getInfoFromMtaYaml(mtaYamlContent []byte, filePath string) (*mtaYaml, error) { 211 var mtaYaml mtaYaml 212 err := yaml.Unmarshal(mtaYamlContent, &mtaYaml) 213 if err != nil { 214 // Eat the original error as it is unhelpful and confusingly mentions JSON, while the 215 // user thinks it should parse YAML (it is transposed by the implementation). 216 return nil, fmt.Errorf("failed to parse contents of the project descriptor file '%s'", 217 filePath) 218 } 219 return &mtaYaml, nil 220 } 221 222 func createMavenExecuteOptions(options *nexusUploadOptions) maven.ExecuteOptions { 223 mavenOptions := maven.ExecuteOptions{ 224 ReturnStdout: false, 225 M2Path: options.M2Path, 226 GlobalSettingsFile: options.GlobalSettingsFile, 227 } 228 return mavenOptions 229 } 230 231 const settingsServerID = "artifact.deployment.nexus" 232 233 const nexusMavenSettings = `<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd"> 234 <servers> 235 <server> 236 <id>artifact.deployment.nexus</id> 237 <username>${env.NEXUS_username}</username> 238 <password>${env.NEXUS_password}</password> 239 </server> 240 </servers> 241 </settings> 242 ` 243 244 const settingsPath = ".pipeline/nexusMavenSettings.xml" 245 246 func setupNexusCredentialsSettingsFile(utils nexusUploadUtils, options *nexusUploadOptions, 247 mavenOptions *maven.ExecuteOptions) (string, error) { 248 if options.Username == "" || options.Password == "" { 249 return "", nil 250 } 251 252 err := utils.FileWrite(settingsPath, []byte(nexusMavenSettings), os.ModePerm) 253 if err != nil { 254 return "", fmt.Errorf("failed to write maven settings file to '%s': %w", settingsPath, err) 255 } 256 257 log.Entry().Debugf("Writing nexus credentials to environment") 258 utils.SetEnv([]string{"NEXUS_username=" + options.Username, "NEXUS_password=" + options.Password}) 259 260 mavenOptions.ProjectSettingsFile = settingsPath 261 mavenOptions.Defines = append(mavenOptions.Defines, "-DrepositoryId="+settingsServerID) 262 return settingsPath, nil 263 } 264 265 type artifactDefines struct { 266 file string 267 packaging string 268 files string 269 classifiers string 270 types string 271 } 272 273 const deployGoal = "org.apache.maven.plugins:maven-deploy-plugin:2.8.2:deploy-file" 274 275 func uploadArtifacts(utils nexusUploadUtils, uploader nexus.Uploader, options *nexusUploadOptions, 276 generatePOM bool) error { 277 if uploader.GetGroupID() == "" { 278 return fmt.Errorf("no group ID was provided, or could be established from project files") 279 } 280 281 artifacts := uploader.GetArtifacts() 282 if len(artifacts) == 0 { 283 return errors.New("no artifacts to upload") 284 } 285 286 var defines []string 287 defines = append(defines, "-Durl="+uploader.GetNexusURLProtocol()+"://"+uploader.GetMavenRepoURL()) 288 defines = append(defines, "-DgroupId="+uploader.GetGroupID()) 289 defines = append(defines, "-Dversion="+uploader.GetArtifactsVersion()) 290 defines = append(defines, "-DartifactId="+uploader.GetArtifactsID()) 291 292 mavenOptions := createMavenExecuteOptions(options) 293 mavenOptions.Goals = []string{deployGoal} 294 mavenOptions.Defines = defines 295 296 settingsFile, err := setupNexusCredentialsSettingsFile(utils, options, &mavenOptions) 297 if err != nil { 298 return fmt.Errorf("writing credential settings for maven failed: %w", err) 299 } 300 if settingsFile != "" { 301 defer func() { _ = utils.FileRemove(settingsFile) }() 302 } 303 304 // iterate over the artifact descriptions, the first one is the main artifact, the following ones are 305 // sub-artifacts. 306 var d artifactDefines 307 for i, artifact := range artifacts { 308 if i == 0 { 309 d.file = artifact.File 310 d.packaging = artifact.Type 311 } else { 312 // Note: It is important to append the comma, even when the list is empty 313 // or the appended item is empty. So classifiers could end up like ",,classes". 314 // This is needed to match the third classifier "classes" to the third sub-artifact. 315 d.files = appendItemToString(d.files, artifact.File, i == 1) 316 d.classifiers = appendItemToString(d.classifiers, artifact.Classifier, i == 1) 317 d.types = appendItemToString(d.types, artifact.Type, i == 1) 318 } 319 } 320 321 err = uploadArtifactsBundle(d, generatePOM, mavenOptions, utils) 322 if err != nil { 323 return fmt.Errorf("uploading artifacts for ID '%s' failed: %w", uploader.GetArtifactsID(), err) 324 } 325 uploader.Clear() 326 return nil 327 } 328 329 // appendItemToString appends a comma this is not the first item, regardless of whether 330 // list or item are empty. 331 func appendItemToString(list, item string, first bool) string { 332 if !first { 333 list += "," 334 } 335 return list + item 336 } 337 338 func uploadArtifactsBundle(d artifactDefines, generatePOM bool, mavenOptions maven.ExecuteOptions, 339 utils nexusUploadUtils) error { 340 if d.file == "" { 341 return fmt.Errorf("no file specified") 342 } 343 344 var defines []string 345 346 defines = append(defines, "-Dfile="+d.file) 347 defines = append(defines, "-Dpackaging="+d.packaging) 348 if !generatePOM { 349 defines = append(defines, "-DgeneratePom=false") 350 } 351 352 if len(d.files) > 0 { 353 defines = append(defines, "-Dfiles="+d.files) 354 defines = append(defines, "-Dclassifiers="+d.classifiers) 355 defines = append(defines, "-Dtypes="+d.types) 356 } 357 358 mavenOptions.Defines = append(mavenOptions.Defines, defines...) 359 _, err := maven.Execute(&mavenOptions, utils) 360 return err 361 } 362 363 func addArtifact(utils nexusUploadUtils, uploader nexus.Uploader, filePath, classifier, fileType string) error { 364 exists, _ := utils.FileExists(filePath) 365 if !exists { 366 return fmt.Errorf("artifact file not found '%s'", filePath) 367 } 368 artifact := nexus.ArtifactDescription{ 369 File: filePath, 370 Type: fileType, 371 Classifier: classifier, 372 } 373 return uploader.AddArtifact(artifact) 374 } 375 376 var errPomNotFound = errors.New("pom.xml not found") 377 378 func uploadMaven(utils nexusUploadUtils, uploader nexus.Uploader, options *nexusUploadOptions) error { 379 pomFiles, _ := utils.Glob("**/pom.xml") 380 if len(pomFiles) == 0 { 381 return errPomNotFound 382 } 383 384 for _, pomFile := range pomFiles { 385 parentDir := filepath.Dir(pomFile) 386 if parentDir == "integration-tests" || parentDir == "unit-tests" { 387 continue 388 } 389 err := uploadMavenArtifacts(utils, uploader, options, parentDir, filepath.Join(parentDir, "target")) 390 if err != nil { 391 return err 392 } 393 } 394 return nil 395 } 396 397 func uploadMavenArtifacts(utils nexusUploadUtils, uploader nexus.Uploader, options *nexusUploadOptions, 398 pomPath, targetFolder string) error { 399 pomFile := composeFilePath(pomPath, "pom", "xml") 400 401 evaluateOptions := &maven.EvaluateOptions{ 402 PomPath: pomFile, 403 GlobalSettingsFile: options.GlobalSettingsFile, 404 M2Path: options.M2Path, 405 } 406 407 packaging, _ := utils.evaluate(evaluateOptions, "project.packaging") 408 if packaging == "" { 409 packaging = "jar" 410 } 411 if packaging != "pom" { 412 // Ignore this module if there is no 'target' folder 413 hasTarget, _ := utils.DirExists(targetFolder) 414 if !hasTarget { 415 log.Entry().Warnf("Ignoring module '%s' as it has no 'target' folder", pomPath) 416 return nil 417 } 418 } 419 groupID, _ := utils.evaluate(evaluateOptions, "project.groupId") 420 if groupID == "" { 421 groupID = options.GroupID 422 } 423 artifactID, err := utils.evaluate(evaluateOptions, "project.artifactId") 424 var artifactsVersion string 425 if err == nil { 426 artifactsVersion, err = utils.evaluate(evaluateOptions, "project.version") 427 } 428 if err == nil { 429 err = uploader.SetInfo(groupID, artifactID, artifactsVersion) 430 } 431 var finalBuildName string 432 if err == nil { 433 finalBuildName, _ = utils.evaluate(evaluateOptions, "project.build.finalName") 434 if finalBuildName == "" { 435 // Fallback to composing final build name, see http://maven.apache.org/pom.html#BaseBuild_Element 436 finalBuildName = artifactID + "-" + artifactsVersion 437 } 438 } 439 if err == nil { 440 err = addArtifact(utils, uploader, pomFile, "", "pom") 441 } 442 if err == nil && packaging != "pom" { 443 err = addMavenTargetArtifacts(utils, uploader, pomFile, targetFolder, finalBuildName, packaging) 444 } 445 if err == nil { 446 err = uploadArtifacts(utils, uploader, options, true) 447 } 448 return err 449 } 450 451 func addMavenTargetArtifacts(utils nexusUploadUtils, uploader nexus.Uploader, pomFile, targetFolder, finalBuildName, packaging string) error { 452 fileTypes := []string{packaging} 453 if packaging != "jar" { 454 // Try to find additional artifacts with a classifier 455 fileTypes = append(fileTypes, "jar") 456 } 457 458 for _, fileType := range fileTypes { 459 pattern := targetFolder + "/*." + fileType 460 matches, _ := utils.Glob(pattern) 461 if len(matches) == 0 && fileType == packaging { 462 return fmt.Errorf("target artifact not found for packaging '%s'", packaging) 463 } 464 log.Entry().Debugf("Glob matches for %s: %s", pattern, strings.Join(matches, ", ")) 465 466 prefix := filepath.Join(targetFolder, finalBuildName) + "-" 467 suffix := "." + fileType 468 for _, filename := range matches { 469 classifier := "" 470 temp := filename 471 if strings.HasPrefix(temp, prefix) && strings.HasSuffix(temp, suffix) { 472 temp = strings.TrimPrefix(temp, prefix) 473 temp = strings.TrimSuffix(temp, suffix) 474 classifier = temp 475 } 476 err := addArtifact(utils, uploader, filename, classifier, fileType) 477 if err != nil { 478 return err 479 } 480 } 481 } 482 return nil 483 } 484 485 func composeFilePath(folder, name, extension string) string { 486 fileName := name + "." + extension 487 return filepath.Join(folder, fileName) 488 }