github.com/SAP/jenkins-library@v1.362.0/cmd/piper.go (about) 1 package cmd 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io" 7 "os" 8 "path/filepath" 9 "reflect" 10 "strconv" 11 "strings" 12 13 "github.com/SAP/jenkins-library/pkg/config" 14 "github.com/SAP/jenkins-library/pkg/log" 15 "github.com/SAP/jenkins-library/pkg/orchestrator" 16 "github.com/SAP/jenkins-library/pkg/piperutils" 17 "github.com/pkg/errors" 18 "github.com/spf13/cobra" 19 ) 20 21 // GeneralConfigOptions contains all global configuration options for piper binary 22 type GeneralConfigOptions struct { 23 GitHubAccessTokens map[string]string // map of tokens with url as key in order to maintain url-specific tokens 24 CorrelationID string 25 CustomConfig string 26 GitHubTokens []string // list of entries in form of <server>:<token> to allow token authentication for downloading config / defaults 27 DefaultConfig []string //ordered list of Piper default configurations. Can be filePath or ENV containing JSON in format 'ENV:MY_ENV_VAR' 28 IgnoreCustomDefaults bool 29 ParametersJSON string 30 EnvRootPath string 31 NoTelemetry bool 32 StageName string 33 StepConfigJSON string 34 StepMetadata string //metadata to be considered, can be filePath or ENV containing JSON in format 'ENV:MY_ENV_VAR' 35 StepName string 36 Verbose bool 37 LogFormat string 38 VaultRoleID string 39 VaultRoleSecretID string 40 VaultToken string 41 VaultServerURL string 42 VaultNamespace string 43 VaultPath string 44 HookConfig HookConfiguration 45 MetaDataResolver func() map[string]config.StepData 46 GCPJsonKeyFilePath string 47 GCSFolderPath string 48 GCSBucketId string 49 GCSSubFolder string 50 } 51 52 // HookConfiguration contains the configuration for supported hooks, so far Sentry and Splunk are supported. 53 type HookConfiguration struct { 54 SentryConfig SentryConfiguration `json:"sentry,omitempty"` 55 SplunkConfig SplunkConfiguration `json:"splunk,omitempty"` 56 PendoConfig PendoConfiguration `json:"pendo,omitempty"` 57 OIDCConfig OIDCConfiguration `json:"oidc,omitempty"` 58 } 59 60 // SentryConfiguration defines the configuration options for the Sentry logging system 61 type SentryConfiguration struct { 62 Dsn string `json:"dsn,omitempty"` 63 } 64 65 // SplunkConfiguration defines the configuration options for the Splunk logging system 66 type SplunkConfiguration struct { 67 Dsn string `json:"dsn,omitempty"` 68 Token string `json:"token,omitempty"` 69 Index string `json:"index,omitempty"` 70 SendLogs bool `json:"sendLogs"` 71 ProdCriblEndpoint string `json:"prodCriblEndpoint,omitempty"` 72 ProdCriblToken string `json:"prodCriblToken,omitempty"` 73 ProdCriblIndex string `json:"prodCriblIndex,omitempty"` 74 } 75 76 type PendoConfiguration struct { 77 Token string `json:"token,omitempty"` 78 } 79 80 // OIDCConfiguration defines the configuration options for the OpenID Connect authentication system 81 type OIDCConfiguration struct { 82 RoleID string `json:",roleID,omitempty"` 83 } 84 85 var rootCmd = &cobra.Command{ 86 Use: "piper", 87 Short: "Executes CI/CD steps from project 'Piper' ", 88 Long: ` 89 This project 'Piper' binary provides a CI/CD step library. 90 It contains many steps which can be used within CI/CD systems as well as directly on e.g. a developer's machine. 91 `, 92 } 93 94 // GeneralConfig contains global configuration flags for piper binary 95 var GeneralConfig GeneralConfigOptions 96 97 // Execute is the starting point of the piper command line tool 98 func Execute() { 99 log.Entry().Infof("Version %s", GitCommit) 100 101 rootCmd.AddCommand(GcpPublishEventCommand()) 102 rootCmd.AddCommand(ArtifactPrepareVersionCommand()) 103 rootCmd.AddCommand(ConfigCommand()) 104 rootCmd.AddCommand(DefaultsCommand()) 105 rootCmd.AddCommand(ContainerSaveImageCommand()) 106 rootCmd.AddCommand(CommandLineCompletionCommand()) 107 rootCmd.AddCommand(VersionCommand()) 108 rootCmd.AddCommand(DetectExecuteScanCommand()) 109 rootCmd.AddCommand(HadolintExecuteCommand()) 110 rootCmd.AddCommand(KarmaExecuteTestsCommand()) 111 rootCmd.AddCommand(UiVeri5ExecuteTestsCommand()) 112 rootCmd.AddCommand(SonarExecuteScanCommand()) 113 rootCmd.AddCommand(KubernetesDeployCommand()) 114 rootCmd.AddCommand(HelmExecuteCommand()) 115 rootCmd.AddCommand(XsDeployCommand()) 116 rootCmd.AddCommand(GithubCheckBranchProtectionCommand()) 117 rootCmd.AddCommand(GithubCommentIssueCommand()) 118 rootCmd.AddCommand(GithubCreateIssueCommand()) 119 rootCmd.AddCommand(GithubCreatePullRequestCommand()) 120 rootCmd.AddCommand(GithubPublishReleaseCommand()) 121 rootCmd.AddCommand(GithubSetCommitStatusCommand()) 122 rootCmd.AddCommand(GitopsUpdateDeploymentCommand()) 123 rootCmd.AddCommand(CloudFoundryDeleteServiceCommand()) 124 rootCmd.AddCommand(AbapEnvironmentPullGitRepoCommand()) 125 rootCmd.AddCommand(AbapEnvironmentCloneGitRepoCommand()) 126 rootCmd.AddCommand(AbapEnvironmentCheckoutBranchCommand()) 127 rootCmd.AddCommand(AbapEnvironmentCreateTagCommand()) 128 rootCmd.AddCommand(AbapEnvironmentCreateSystemCommand()) 129 rootCmd.AddCommand(CheckmarxExecuteScanCommand()) 130 rootCmd.AddCommand(CheckmarxOneExecuteScanCommand()) 131 rootCmd.AddCommand(FortifyExecuteScanCommand()) 132 rootCmd.AddCommand(CodeqlExecuteScanCommand()) 133 rootCmd.AddCommand(ContrastExecuteScanCommand()) 134 rootCmd.AddCommand(CredentialdiggerScanCommand()) 135 rootCmd.AddCommand(MtaBuildCommand()) 136 rootCmd.AddCommand(ProtecodeExecuteScanCommand()) 137 rootCmd.AddCommand(MavenExecuteCommand()) 138 rootCmd.AddCommand(CloudFoundryCreateServiceKeyCommand()) 139 rootCmd.AddCommand(MavenBuildCommand()) 140 rootCmd.AddCommand(MavenExecuteIntegrationCommand()) 141 rootCmd.AddCommand(MavenExecuteStaticCodeChecksCommand()) 142 rootCmd.AddCommand(NexusUploadCommand()) 143 rootCmd.AddCommand(AbapEnvironmentPushATCSystemConfigCommand()) 144 rootCmd.AddCommand(AbapEnvironmentRunATCCheckCommand()) 145 rootCmd.AddCommand(NpmExecuteScriptsCommand()) 146 rootCmd.AddCommand(NpmExecuteLintCommand()) 147 rootCmd.AddCommand(GctsCreateRepositoryCommand()) 148 rootCmd.AddCommand(GctsExecuteABAPQualityChecksCommand()) 149 rootCmd.AddCommand(GctsExecuteABAPUnitTestsCommand()) 150 rootCmd.AddCommand(GctsDeployCommand()) 151 rootCmd.AddCommand(MalwareExecuteScanCommand()) 152 rootCmd.AddCommand(CloudFoundryCreateServiceCommand()) 153 rootCmd.AddCommand(CloudFoundryDeployCommand()) 154 rootCmd.AddCommand(GctsRollbackCommand()) 155 rootCmd.AddCommand(WhitesourceExecuteScanCommand()) 156 rootCmd.AddCommand(GctsCloneRepositoryCommand()) 157 rootCmd.AddCommand(JsonApplyPatchCommand()) 158 rootCmd.AddCommand(KanikoExecuteCommand()) 159 rootCmd.AddCommand(CnbBuildCommand()) 160 rootCmd.AddCommand(AbapEnvironmentBuildCommand()) 161 rootCmd.AddCommand(AbapEnvironmentAssemblePackagesCommand()) 162 rootCmd.AddCommand(AbapAddonAssemblyKitCheckCVsCommand()) 163 rootCmd.AddCommand(AbapAddonAssemblyKitCheckPVCommand()) 164 rootCmd.AddCommand(AbapAddonAssemblyKitCheckCommand()) 165 rootCmd.AddCommand(AbapAddonAssemblyKitCreateTargetVectorCommand()) 166 rootCmd.AddCommand(AbapAddonAssemblyKitPublishTargetVectorCommand()) 167 rootCmd.AddCommand(AbapAddonAssemblyKitRegisterPackagesCommand()) 168 rootCmd.AddCommand(AbapAddonAssemblyKitReleasePackagesCommand()) 169 rootCmd.AddCommand(AbapAddonAssemblyKitReserveNextPackagesCommand()) 170 rootCmd.AddCommand(CloudFoundryCreateSpaceCommand()) 171 rootCmd.AddCommand(CloudFoundryDeleteSpaceCommand()) 172 rootCmd.AddCommand(VaultRotateSecretIdCommand()) 173 rootCmd.AddCommand(IsChangeInDevelopmentCommand()) 174 rootCmd.AddCommand(TransportRequestUploadCTSCommand()) 175 rootCmd.AddCommand(TransportRequestUploadRFCCommand()) 176 rootCmd.AddCommand(NewmanExecuteCommand()) 177 rootCmd.AddCommand(IntegrationArtifactDeployCommand()) 178 rootCmd.AddCommand(TransportRequestUploadSOLMANCommand()) 179 rootCmd.AddCommand(IntegrationArtifactUpdateConfigurationCommand()) 180 rootCmd.AddCommand(IntegrationArtifactGetMplStatusCommand()) 181 rootCmd.AddCommand(IntegrationArtifactGetServiceEndpointCommand()) 182 rootCmd.AddCommand(IntegrationArtifactDownloadCommand()) 183 rootCmd.AddCommand(AbapEnvironmentAssembleConfirmCommand()) 184 rootCmd.AddCommand(IntegrationArtifactUploadCommand()) 185 rootCmd.AddCommand(IntegrationArtifactTriggerIntegrationTestCommand()) 186 rootCmd.AddCommand(IntegrationArtifactUnDeployCommand()) 187 rootCmd.AddCommand(IntegrationArtifactResourceCommand()) 188 rootCmd.AddCommand(TerraformExecuteCommand()) 189 rootCmd.AddCommand(ContainerExecuteStructureTestsCommand()) 190 rootCmd.AddCommand(GaugeExecuteTestsCommand()) 191 rootCmd.AddCommand(BatsExecuteTestsCommand()) 192 rootCmd.AddCommand(PipelineCreateScanSummaryCommand()) 193 rootCmd.AddCommand(TransportRequestDocIDFromGitCommand()) 194 rootCmd.AddCommand(TransportRequestReqIDFromGitCommand()) 195 rootCmd.AddCommand(WritePipelineEnv()) 196 rootCmd.AddCommand(ReadPipelineEnv()) 197 rootCmd.AddCommand(InfluxWriteDataCommand()) 198 rootCmd.AddCommand(AbapEnvironmentRunAUnitTestCommand()) 199 rootCmd.AddCommand(CheckStepActiveCommand()) 200 rootCmd.AddCommand(GolangBuildCommand()) 201 rootCmd.AddCommand(ShellExecuteCommand()) 202 rootCmd.AddCommand(ApiProxyDownloadCommand()) 203 rootCmd.AddCommand(ApiKeyValueMapDownloadCommand()) 204 rootCmd.AddCommand(ApiProviderDownloadCommand()) 205 rootCmd.AddCommand(ApiProxyUploadCommand()) 206 rootCmd.AddCommand(GradleExecuteBuildCommand()) 207 rootCmd.AddCommand(ApiKeyValueMapUploadCommand()) 208 rootCmd.AddCommand(PythonBuildCommand()) 209 rootCmd.AddCommand(AzureBlobUploadCommand()) 210 rootCmd.AddCommand(AwsS3UploadCommand()) 211 rootCmd.AddCommand(ApiProxyListCommand()) 212 rootCmd.AddCommand(AnsSendEventCommand()) 213 rootCmd.AddCommand(ApiProviderListCommand()) 214 rootCmd.AddCommand(TmsUploadCommand()) 215 rootCmd.AddCommand(TmsExportCommand()) 216 rootCmd.AddCommand(IntegrationArtifactTransportCommand()) 217 rootCmd.AddCommand(AscAppUploadCommand()) 218 rootCmd.AddCommand(AbapLandscapePortalUpdateAddOnProductCommand()) 219 rootCmd.AddCommand(ImagePushToRegistryCommand()) 220 221 addRootFlags(rootCmd) 222 223 if err := rootCmd.Execute(); err != nil { 224 log.SetErrorCategory(log.ErrorConfiguration) 225 log.Entry().WithError(err).Fatal("configuration error") 226 } 227 } 228 229 func addRootFlags(rootCmd *cobra.Command) { 230 provider, err := orchestrator.GetOrchestratorConfigProvider(nil) 231 if err != nil { 232 log.Entry().Error(err) 233 provider = &orchestrator.UnknownOrchestratorConfigProvider{} 234 } 235 236 rootCmd.PersistentFlags().StringVar(&GeneralConfig.CorrelationID, "correlationID", provider.BuildURL(), "ID for unique identification of a pipeline run") 237 rootCmd.PersistentFlags().StringVar(&GeneralConfig.CustomConfig, "customConfig", ".pipeline/config.yml", "Path to the pipeline configuration file") 238 rootCmd.PersistentFlags().StringSliceVar(&GeneralConfig.GitHubTokens, "gitHubTokens", AccessTokensFromEnvJSON(os.Getenv("PIPER_gitHubTokens")), "List of entries in form of <hostname>:<token> to allow GitHub token authentication for downloading config / defaults") 239 rootCmd.PersistentFlags().StringSliceVar(&GeneralConfig.DefaultConfig, "defaultConfig", []string{".pipeline/defaults.yaml"}, "Default configurations, passed as path to yaml file") 240 rootCmd.PersistentFlags().BoolVar(&GeneralConfig.IgnoreCustomDefaults, "ignoreCustomDefaults", false, "Disables evaluation of the parameter 'customDefaults' in the pipeline configuration file") 241 rootCmd.PersistentFlags().StringVar(&GeneralConfig.ParametersJSON, "parametersJSON", os.Getenv("PIPER_parametersJSON"), "Parameters to be considered in JSON format") 242 rootCmd.PersistentFlags().StringVar(&GeneralConfig.EnvRootPath, "envRootPath", ".pipeline", "Root path to Piper pipeline shared environments") 243 rootCmd.PersistentFlags().StringVar(&GeneralConfig.StageName, "stageName", "", "Name of the stage for which configuration should be included") 244 rootCmd.PersistentFlags().StringVar(&GeneralConfig.StepConfigJSON, "stepConfigJSON", os.Getenv("PIPER_stepConfigJSON"), "Step configuration in JSON format") 245 rootCmd.PersistentFlags().BoolVar(&GeneralConfig.NoTelemetry, "noTelemetry", false, "Disables telemetry reporting") 246 rootCmd.PersistentFlags().BoolVarP(&GeneralConfig.Verbose, "verbose", "v", false, "verbose output") 247 rootCmd.PersistentFlags().StringVar(&GeneralConfig.LogFormat, "logFormat", "default", "Log format to use. Options: default, timestamp, plain, full.") 248 rootCmd.PersistentFlags().StringVar(&GeneralConfig.VaultServerURL, "vaultServerUrl", "", "The Vault server which should be used to fetch credentials") 249 rootCmd.PersistentFlags().StringVar(&GeneralConfig.VaultNamespace, "vaultNamespace", "", "The Vault namespace which should be used to fetch credentials") 250 rootCmd.PersistentFlags().StringVar(&GeneralConfig.VaultPath, "vaultPath", "", "The path which should be used to fetch credentials") 251 rootCmd.PersistentFlags().StringVar(&GeneralConfig.GCPJsonKeyFilePath, "gcpJsonKeyFilePath", "", "File path to Google Cloud Platform JSON key file") 252 rootCmd.PersistentFlags().StringVar(&GeneralConfig.GCSFolderPath, "gcsFolderPath", "", "GCS folder path. One of the components of GCS target folder") 253 rootCmd.PersistentFlags().StringVar(&GeneralConfig.GCSBucketId, "gcsBucketId", "", "Bucket name for Google Cloud Storage") 254 rootCmd.PersistentFlags().StringVar(&GeneralConfig.GCSSubFolder, "gcsSubFolder", "", "Used to logically separate results of the same step result type") 255 256 } 257 258 // ResolveAccessTokens reads a list of tokens in format host:token passed via command line 259 // and transfers this into a map as a more consumable format. 260 func ResolveAccessTokens(tokenList []string) map[string]string { 261 tokenMap := map[string]string{} 262 for _, tokenEntry := range tokenList { 263 log.Entry().Debugf("processing token %v", tokenEntry) 264 parts := strings.Split(tokenEntry, ":") 265 if len(parts) != 2 { 266 log.Entry().Warningf("wrong format for access token %v", tokenEntry) 267 } else { 268 tokenMap[parts[0]] = parts[1] 269 } 270 } 271 return tokenMap 272 } 273 274 // AccessTokensFromEnvJSON resolves access tokens when passed as JSON in an environment variable 275 func AccessTokensFromEnvJSON(env string) []string { 276 accessTokens := []string{} 277 if len(env) == 0 { 278 return accessTokens 279 } 280 err := json.Unmarshal([]byte(env), &accessTokens) 281 if err != nil { 282 log.Entry().Infof("Token json '%v' has wrong format.", env) 283 } 284 return accessTokens 285 } 286 287 // initStageName initializes GeneralConfig.StageName from either GeneralConfig.ParametersJSON 288 // or the environment variable (orchestrator specific), unless it has been provided as command line option. 289 // Log output needs to be suppressed via outputToLog by the getConfig step. 290 func initStageName(outputToLog bool) { 291 var stageNameSource string 292 if outputToLog { 293 defer func() { 294 log.Entry().Debugf("Using stageName '%s' from %s", GeneralConfig.StageName, stageNameSource) 295 }() 296 } 297 298 if GeneralConfig.StageName != "" { 299 // Means it was given as command line argument and has the highest precedence 300 stageNameSource = "command line arguments" 301 return 302 } 303 304 // Use stageName from ENV as fall-back, for when extracting it from parametersJSON fails below 305 provider, err := orchestrator.GetOrchestratorConfigProvider(nil) 306 if err != nil { 307 log.Entry().WithError(err).Warning("Cannot infer stage name from CI environment") 308 } else { 309 stageNameSource = "env variable" 310 GeneralConfig.StageName = provider.StageName() 311 } 312 313 if len(GeneralConfig.ParametersJSON) == 0 { 314 return 315 } 316 317 var params map[string]interface{} 318 err = json.Unmarshal([]byte(GeneralConfig.ParametersJSON), ¶ms) 319 if err != nil { 320 if outputToLog { 321 log.Entry().Infof("Failed to extract 'stageName' from parametersJSON: %v", err) 322 } 323 return 324 } 325 326 stageName, hasKey := params["stageName"] 327 if !hasKey { 328 return 329 } 330 331 if stageNameString, ok := stageName.(string); ok && stageNameString != "" { 332 stageNameSource = "parametersJSON" 333 GeneralConfig.StageName = stageNameString 334 } 335 } 336 337 // PrepareConfig reads step configuration from various sources and merges it (defaults, config file, flags, ...) 338 func PrepareConfig(cmd *cobra.Command, metadata *config.StepData, stepName string, options interface{}, openFile func(s string, t map[string]string) (io.ReadCloser, error)) error { 339 340 log.SetFormatter(GeneralConfig.LogFormat) 341 342 initStageName(true) 343 344 filters := metadata.GetParameterFilters() 345 346 // add telemetry parameter "collectTelemetryData" to ALL, GENERAL and PARAMETER filters 347 filters.All = append(filters.All, "collectTelemetryData") 348 filters.General = append(filters.General, "collectTelemetryData") 349 filters.Parameters = append(filters.Parameters, "collectTelemetryData") 350 351 envParams := metadata.GetResourceParameters(GeneralConfig.EnvRootPath, "commonPipelineEnvironment") 352 reportingEnvParams := config.ReportingParameters.GetResourceParameters(GeneralConfig.EnvRootPath, "commonPipelineEnvironment") 353 resourceParams := mergeResourceParameters(envParams, reportingEnvParams) 354 355 flagValues := config.AvailableFlagValues(cmd, &filters) 356 357 var myConfig config.Config 358 var stepConfig config.StepConfig 359 360 // add vault credentials so that configuration can be fetched from vault 361 if GeneralConfig.VaultRoleID == "" { 362 GeneralConfig.VaultRoleID = os.Getenv("PIPER_vaultAppRoleID") 363 } 364 if GeneralConfig.VaultRoleSecretID == "" { 365 GeneralConfig.VaultRoleSecretID = os.Getenv("PIPER_vaultAppRoleSecretID") 366 } 367 if GeneralConfig.VaultToken == "" { 368 GeneralConfig.VaultToken = os.Getenv("PIPER_vaultToken") 369 } 370 myConfig.SetVaultCredentials(GeneralConfig.VaultRoleID, GeneralConfig.VaultRoleSecretID, GeneralConfig.VaultToken) 371 372 if len(GeneralConfig.StepConfigJSON) != 0 { 373 // ignore config & defaults in favor of passed stepConfigJSON 374 stepConfig = config.GetStepConfigWithJSON(flagValues, GeneralConfig.StepConfigJSON, filters) 375 log.Entry().Infof("Project config: passed via JSON") 376 log.Entry().Infof("Project defaults: passed via JSON") 377 } else { 378 // use config & defaults 379 var customConfig io.ReadCloser 380 var err error 381 //accept that config file and defaults cannot be loaded since both are not mandatory here 382 { 383 projectConfigFile := getProjectConfigFile(GeneralConfig.CustomConfig) 384 if exists, err := piperutils.FileExists(projectConfigFile); exists { 385 log.Entry().Debugf("Project config: '%s'", projectConfigFile) 386 if customConfig, err = openFile(projectConfigFile, GeneralConfig.GitHubAccessTokens); err != nil { 387 return errors.Wrapf(err, "Cannot read '%s'", projectConfigFile) 388 } 389 } else { 390 log.Entry().Infof("Project config: NONE ('%s' does not exist)", projectConfigFile) 391 customConfig = nil 392 } 393 } 394 var defaultConfig []io.ReadCloser 395 if len(GeneralConfig.DefaultConfig) == 0 { 396 log.Entry().Info("Project defaults: NONE") 397 } 398 for _, projectDefaultFile := range GeneralConfig.DefaultConfig { 399 fc, err := openFile(projectDefaultFile, GeneralConfig.GitHubAccessTokens) 400 // only create error for non-default values 401 if err != nil { 402 if projectDefaultFile != ".pipeline/defaults.yaml" { 403 log.Entry().Debugf("Project defaults: '%s'", projectDefaultFile) 404 return errors.Wrapf(err, "Cannot read '%s'", projectDefaultFile) 405 } 406 } else { 407 log.Entry().Debugf("Project defaults: '%s'", projectDefaultFile) 408 defaultConfig = append(defaultConfig, fc) 409 } 410 } 411 stepConfig, err = myConfig.GetStepConfig(flagValues, GeneralConfig.ParametersJSON, customConfig, defaultConfig, GeneralConfig.IgnoreCustomDefaults, filters, *metadata, resourceParams, GeneralConfig.StageName, stepName) 412 if verbose, ok := stepConfig.Config["verbose"].(bool); ok && verbose { 413 log.SetVerbose(verbose) 414 GeneralConfig.Verbose = verbose 415 } else if !ok && stepConfig.Config["verbose"] != nil { 416 log.Entry().Warnf("invalid value for parameter verbose: '%v'", stepConfig.Config["verbose"]) 417 } 418 if err != nil { 419 return errors.Wrap(err, "retrieving step configuration failed") 420 } 421 } 422 423 stepConfig.Config = checkTypes(stepConfig.Config, options) 424 confJSON, _ := json.Marshal(stepConfig.Config) 425 _ = json.Unmarshal(confJSON, &options) 426 427 config.MarkFlagsWithValue(cmd, stepConfig) 428 429 retrieveHookConfig(stepConfig.HookConfig, &GeneralConfig.HookConfig) 430 431 if GeneralConfig.GCPJsonKeyFilePath == "" { 432 GeneralConfig.GCPJsonKeyFilePath, _ = stepConfig.Config["gcpJsonKeyFilePath"].(string) 433 } 434 if GeneralConfig.GCSFolderPath == "" { 435 GeneralConfig.GCSFolderPath, _ = stepConfig.Config["gcsFolderPath"].(string) 436 } 437 if GeneralConfig.GCSBucketId == "" { 438 GeneralConfig.GCSBucketId, _ = stepConfig.Config["gcsBucketId"].(string) 439 } 440 if GeneralConfig.GCSSubFolder == "" { 441 GeneralConfig.GCSSubFolder, _ = stepConfig.Config["gcsSubFolder"].(string) 442 } 443 return nil 444 } 445 446 func retrieveHookConfig(source map[string]interface{}, target *HookConfiguration) { 447 if source != nil { 448 log.Entry().Debug("Retrieving hook configuration") 449 b, err := json.Marshal(source) 450 if err != nil { 451 log.Entry().Warningf("Failed to marshal source hook configuration: %v", err) 452 } 453 err = json.Unmarshal(b, target) 454 if err != nil { 455 log.Entry().Warningf("Failed to retrieve hook configuration: %v", err) 456 } 457 } 458 } 459 460 var errIncompatibleTypes = fmt.Errorf("incompatible types") 461 462 func checkTypes(config map[string]interface{}, options interface{}) map[string]interface{} { 463 optionsType := getStepOptionsStructType(options) 464 465 for paramName := range config { 466 optionsField := findStructFieldByJSONTag(paramName, optionsType) 467 if optionsField == nil { 468 continue 469 } 470 471 if config[paramName] == nil { 472 // There is a key, but no value. This can result from merging values from the CPE. 473 continue 474 } 475 476 paramValueType := reflect.ValueOf(config[paramName]) 477 if optionsField.Type.Kind() == paramValueType.Kind() { 478 // Types already match, nothing to do 479 continue 480 } 481 482 var typeError error = nil 483 484 switch paramValueType.Kind() { 485 case reflect.String: 486 typeError = convertValueFromString(config, optionsField, paramName, paramValueType.String()) 487 case reflect.Float32, reflect.Float64: 488 typeError = convertValueFromFloat(config, optionsField, paramName, paramValueType.Float()) 489 case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: 490 typeError = convertValueFromInt(config, optionsField, paramName, paramValueType.Int()) 491 default: 492 log.Entry().Warnf("Config value for '%s' is of unexpected type %s, expected %s. "+ 493 "The value may be ignored as a result. To avoid any risk, specify this value with explicit type.", 494 paramName, paramValueType.Kind(), optionsField.Type.Kind()) 495 } 496 497 if typeError != nil { 498 typeError = fmt.Errorf("config value for '%s' is of unexpected type %s, expected %s: %w", 499 paramName, paramValueType.Kind(), optionsField.Type.Kind(), typeError) 500 log.SetErrorCategory(log.ErrorConfiguration) 501 log.Entry().WithError(typeError).Fatal("type error in configuration") 502 } 503 } 504 return config 505 } 506 507 func convertValueFromString(config map[string]interface{}, optionsField *reflect.StructField, paramName, paramValue string) error { 508 switch optionsField.Type.Kind() { 509 case reflect.Slice, reflect.Array: 510 // Could do automatic conversion for those types in theory, 511 // but that might obscure what really happens in error cases. 512 return fmt.Errorf("expected type to be a list (or slice, or array) but got string") 513 case reflect.Bool: 514 // Sensible to convert strings "true"/"false" to respective boolean values as it is 515 // common practice to write booleans as string in yaml files. 516 paramValue = strings.ToLower(paramValue) 517 if paramValue == "true" { 518 config[paramName] = true 519 return nil 520 } else if paramValue == "false" { 521 config[paramName] = false 522 return nil 523 } 524 } 525 526 return errIncompatibleTypes 527 } 528 529 func convertValueFromFloat(config map[string]interface{}, optionsField *reflect.StructField, paramName string, paramValue float64) error { 530 switch optionsField.Type.Kind() { 531 case reflect.String: 532 val := strconv.FormatFloat(paramValue, 'f', -1, 64) 533 // if Sprinted value and val are equal, we can be pretty sure that the result fits 534 // for very large numbers for example an exponential format is printed 535 if val == fmt.Sprint(paramValue) { 536 config[paramName] = val 537 return nil 538 } 539 // allow float numbers containing a decimal separator 540 if strings.Contains(val, ".") { 541 config[paramName] = val 542 return nil 543 } 544 // if now no decimal separator is available we cannot be sure that the result is correct: 545 // long numbers like e.g. 73554900100200011600 will not be represented correctly after reading the yaml 546 // thus we cannot assume that the string is correct. 547 // short numbers will be handled as int anyway 548 return errIncompatibleTypes 549 case reflect.Float32: 550 config[paramName] = float32(paramValue) 551 return nil 552 case reflect.Float64: 553 config[paramName] = paramValue 554 return nil 555 case reflect.Int: 556 // Treat as type-mismatch only in case the conversion would be lossy. 557 // In that case, the json.Unmarshall() would indeed just drop it, so we want to fail. 558 if float64(int(paramValue)) == paramValue { 559 config[paramName] = int(paramValue) 560 return nil 561 } 562 } 563 564 return errIncompatibleTypes 565 } 566 567 func convertValueFromInt(config map[string]interface{}, optionsField *reflect.StructField, paramName string, paramValue int64) error { 568 switch optionsField.Type.Kind() { 569 case reflect.String: 570 config[paramName] = strconv.FormatInt(paramValue, 10) 571 return nil 572 case reflect.Float32: 573 config[paramName] = float32(paramValue) 574 return nil 575 case reflect.Float64: 576 config[paramName] = float64(paramValue) 577 return nil 578 } 579 580 return errIncompatibleTypes 581 } 582 583 func findStructFieldByJSONTag(tagName string, optionsType reflect.Type) *reflect.StructField { 584 for i := 0; i < optionsType.NumField(); i++ { 585 field := optionsType.Field(i) 586 tag := field.Tag.Get("json") 587 if tagName == tag || tagName+",omitempty" == tag { 588 return &field 589 } 590 } 591 return nil 592 } 593 594 func getStepOptionsStructType(stepOptions interface{}) reflect.Type { 595 typedOptions := reflect.ValueOf(stepOptions) 596 if typedOptions.Kind() == reflect.Ptr { 597 typedOptions = typedOptions.Elem() 598 } 599 return typedOptions.Type() 600 } 601 602 func getProjectConfigFile(name string) string { 603 604 var altName string 605 if ext := filepath.Ext(name); ext == ".yml" { 606 altName = fmt.Sprintf("%v.yaml", strings.TrimSuffix(name, ext)) 607 } else if ext == "yaml" { 608 altName = fmt.Sprintf("%v.yml", strings.TrimSuffix(name, ext)) 609 } 610 611 fileExists, _ := piperutils.FileExists(name) 612 altExists, _ := piperutils.FileExists(altName) 613 614 // configured filename will always take precedence, even if not existing 615 if !fileExists && altExists { 616 return altName 617 } 618 return name 619 } 620 621 func mergeResourceParameters(resParams ...map[string]interface{}) map[string]interface{} { 622 result := make(map[string]interface{}) 623 for _, m := range resParams { 624 for k, v := range m { 625 result[k] = v 626 } 627 } 628 return result 629 }