github.com/jaylevin/jenkins-library@v1.230.4/pkg/config/stepmeta.go (about) 1 package config 2 3 import ( 4 "bytes" 5 "encoding/json" 6 "fmt" 7 "io" 8 "io/ioutil" 9 10 "github.com/SAP/jenkins-library/pkg/log" 11 "github.com/SAP/jenkins-library/pkg/piperenv" 12 13 "github.com/ghodss/yaml" 14 "github.com/pkg/errors" 15 ) 16 17 // StepData defines the metadata for a step, like step descriptions, parameters, ... 18 type StepData struct { 19 Metadata StepMetadata `json:"metadata"` 20 Spec StepSpec `json:"spec"` 21 } 22 23 // StepMetadata defines the metadata for a step, like step descriptions, parameters, ... 24 type StepMetadata struct { 25 Name string `json:"name"` 26 Aliases []Alias `json:"aliases,omitempty"` 27 Description string `json:"description"` 28 LongDescription string `json:"longDescription,omitempty"` 29 } 30 31 // StepSpec defines the spec details for a step, like step inputs, containers, sidecars, ... 32 type StepSpec struct { 33 Inputs StepInputs `json:"inputs,omitempty"` 34 Outputs StepOutputs `json:"outputs,omitempty"` 35 Containers []Container `json:"containers,omitempty"` 36 Sidecars []Container `json:"sidecars,omitempty"` 37 } 38 39 // StepInputs defines the spec details for a step, like step inputs, containers, sidecars, ... 40 type StepInputs struct { 41 Parameters []StepParameters `json:"params"` 42 Resources []StepResources `json:"resources,omitempty"` 43 Secrets []StepSecrets `json:"secrets,omitempty"` 44 } 45 46 // StepParameters defines the parameters for a step 47 type StepParameters struct { 48 Name string `json:"name"` 49 Description string `json:"description"` 50 LongDescription string `json:"longDescription,omitempty"` 51 ResourceRef []ResourceReference `json:"resourceRef,omitempty"` 52 Scope []string `json:"scope"` 53 Type string `json:"type"` 54 Mandatory bool `json:"mandatory,omitempty"` 55 Default interface{} `json:"default,omitempty"` 56 PossibleValues []interface{} `json:"possibleValues,omitempty"` 57 Aliases []Alias `json:"aliases,omitempty"` 58 Conditions []Condition `json:"conditions,omitempty"` 59 Secret bool `json:"secret,omitempty"` 60 MandatoryIf []ParameterDependence `json:"mandatoryIf,omitempty"` 61 DeprecationMessage string `json:"deprecationMessage,omitempty"` 62 } 63 64 type ParameterDependence struct { 65 Name string `json:"name"` 66 Value string `json:"value"` 67 } 68 69 // ResourceReference defines the parameters of a resource reference 70 type ResourceReference struct { 71 Name string `json:"name"` 72 Type string `json:"type,omitempty"` 73 Param string `json:"param,omitempty"` 74 Default string `json:"default,omitempty"` 75 Aliases []Alias `json:"aliases,omitempty"` 76 } 77 78 // Alias defines a step input parameter alias 79 type Alias struct { 80 Name string `json:"name,omitempty"` 81 Deprecated bool `json:"deprecated,omitempty"` 82 } 83 84 // StepResources defines the resources to be provided by the step context, e.g. Jenkins pipeline 85 type StepResources struct { 86 Name string `json:"name"` 87 Description string `json:"description,omitempty"` 88 Type string `json:"type,omitempty"` 89 Parameters []map[string]interface{} `json:"params,omitempty"` 90 Conditions []Condition `json:"conditions,omitempty"` 91 } 92 93 // StepSecrets defines the secrets to be provided by the step context, e.g. Jenkins pipeline 94 type StepSecrets struct { 95 Name string `json:"name"` 96 Description string `json:"description,omitempty"` 97 Type string `json:"type,omitempty"` 98 Aliases []Alias `json:"aliases,omitempty"` 99 } 100 101 // StepOutputs defines the outputs of a step step, typically one or multiple resources 102 type StepOutputs struct { 103 Resources []StepResources `json:"resources,omitempty"` 104 } 105 106 // Container defines an execution container 107 type Container struct { 108 //ToDo: check dockerOptions, dockerVolumeBind, containerPortMappings, sidecarOptions, sidecarVolumeBind 109 Command []string `json:"command"` 110 EnvVars []EnvVar `json:"env"` 111 Image string `json:"image"` 112 ImagePullPolicy string `json:"imagePullPolicy"` 113 Name string `json:"name"` 114 ReadyCommand string `json:"readyCommand"` 115 Shell string `json:"shell"` 116 WorkingDir string `json:"workingDir"` 117 Conditions []Condition `json:"conditions,omitempty"` 118 Options []Option `json:"options,omitempty"` 119 //VolumeMounts []VolumeMount `json:"volumeMounts,omitempty"` 120 } 121 122 // ToDo: Add the missing Volumes part to enable the volume mount completely 123 // VolumeMount defines a mount path 124 // type VolumeMount struct { 125 // MountPath string `json:"mountPath"` 126 // Name string `json:"name"` 127 //} 128 129 // Option defines an docker option 130 type Option struct { 131 Name string `json:"name"` 132 Value string `json:"value"` 133 } 134 135 // EnvVar defines an environment variable 136 type EnvVar struct { 137 Name string `json:"name"` 138 Value string `json:"value"` 139 } 140 141 // Condition defines an condition which decides when the parameter, resource or container is valid 142 type Condition struct { 143 ConditionRef string `json:"conditionRef"` 144 Params []Param `json:"params"` 145 } 146 147 // Param defines the parameters serving as inputs to the condition 148 type Param struct { 149 Name string `json:"name"` 150 Value string `json:"value"` 151 } 152 153 // StepFilters defines the filter parameters for the different sections 154 type StepFilters struct { 155 All []string 156 General []string 157 Stages []string 158 Steps []string 159 Parameters []string 160 Env []string 161 } 162 163 // ReadPipelineStepData loads step definition in yaml format 164 func (m *StepData) ReadPipelineStepData(metadata io.ReadCloser) error { 165 defer metadata.Close() 166 content, err := ioutil.ReadAll(metadata) 167 if err != nil { 168 return errors.Wrapf(err, "error reading %v", metadata) 169 } 170 171 err = yaml.Unmarshal(content, &m) 172 if err != nil { 173 return errors.Wrapf(err, "error unmarshalling: %v", err) 174 } 175 return nil 176 } 177 178 // GetParameterFilters retrieves all scope dependent parameter filters 179 func (m *StepData) GetParameterFilters() StepFilters { 180 filters := StepFilters{All: []string{"verbose"}, General: []string{"verbose"}, Steps: []string{"verbose"}, Stages: []string{"verbose"}, Parameters: []string{"verbose"}} 181 for _, param := range m.Spec.Inputs.Parameters { 182 parameterKeys := []string{param.Name} 183 for _, condition := range param.Conditions { 184 for _, dependentParam := range condition.Params { 185 parameterKeys = append(parameterKeys, dependentParam.Value) 186 } 187 } 188 filters.All = append(filters.All, parameterKeys...) 189 for _, scope := range param.Scope { 190 switch scope { 191 case "GENERAL": 192 filters.General = append(filters.General, parameterKeys...) 193 case "STEPS": 194 filters.Steps = append(filters.Steps, parameterKeys...) 195 case "STAGES": 196 filters.Stages = append(filters.Stages, parameterKeys...) 197 case "PARAMETERS": 198 filters.Parameters = append(filters.Parameters, parameterKeys...) 199 case "ENV": 200 filters.Env = append(filters.Env, parameterKeys...) 201 } 202 } 203 } 204 return filters 205 } 206 207 // GetContextParameterFilters retrieves all scope dependent parameter filters 208 func (m *StepData) GetContextParameterFilters() StepFilters { 209 var filters StepFilters 210 contextFilters := []string{} 211 for _, secret := range m.Spec.Inputs.Secrets { 212 contextFilters = append(contextFilters, secret.Name) 213 } 214 215 if len(m.Spec.Inputs.Resources) > 0 { 216 for _, res := range m.Spec.Inputs.Resources { 217 if res.Type == "stash" { 218 contextFilters = append(contextFilters, "stashContent") 219 break 220 } 221 } 222 } 223 if len(m.Spec.Containers) > 0 { 224 parameterKeys := []string{"containerCommand", "containerShell", "dockerEnvVars", "dockerImage", "dockerName", "dockerOptions", "dockerPullImage", "dockerVolumeBind", "dockerWorkspace", "dockerRegistryUrl", "dockerRegistryCredentialsId"} 225 for _, container := range m.Spec.Containers { 226 for _, condition := range container.Conditions { 227 for _, dependentParam := range condition.Params { 228 parameterKeys = append(parameterKeys, dependentParam.Value) 229 parameterKeys = append(parameterKeys, dependentParam.Name) 230 } 231 } 232 } 233 // ToDo: append dependentParam.Value & dependentParam.Name only according to correct parameter scope and not generally 234 contextFilters = append(contextFilters, parameterKeys...) 235 } 236 if len(m.Spec.Sidecars) > 0 { 237 //ToDo: support fallback for "dockerName" configuration property -> via aliasing? 238 contextFilters = append(contextFilters, []string{"containerName", "containerPortMappings", "dockerName", "sidecarEnvVars", "sidecarImage", "sidecarName", "sidecarOptions", "sidecarPullImage", "sidecarReadyCommand", "sidecarVolumeBind", "sidecarWorkspace"}...) 239 //ToDo: add condition param.Value and param.Name to filter as for Containers 240 } 241 242 contextFilters = addVaultContextParametersFilter(m, contextFilters) 243 244 if len(contextFilters) > 0 { 245 filters.All = append(filters.All, contextFilters...) 246 filters.General = append(filters.General, contextFilters...) 247 filters.Steps = append(filters.Steps, contextFilters...) 248 filters.Stages = append(filters.Stages, contextFilters...) 249 filters.Parameters = append(filters.Parameters, contextFilters...) 250 filters.Env = append(filters.Env, contextFilters...) 251 252 } 253 return filters 254 } 255 256 func addVaultContextParametersFilter(m *StepData, contextFilters []string) []string { 257 contextFilters = append(contextFilters, []string{"vaultAppRoleTokenCredentialsId", 258 "vaultAppRoleSecretTokenCredentialsId", "vaultTokenCredentialsId"}...) 259 return contextFilters 260 } 261 262 // GetContextDefaults retrieves context defaults like container image, name, env vars, resources, ... 263 // It only supports scenarios with one container and optionally one sidecar 264 func (m *StepData) GetContextDefaults(stepName string) (io.ReadCloser, error) { 265 266 //ToDo error handling empty Containers/Sidecars 267 //ToDo handle empty Command 268 root := map[string]interface{}{} 269 if len(m.Spec.Containers) > 0 { 270 for _, container := range m.Spec.Containers { 271 key := "" 272 conditionParam := "" 273 if len(container.Conditions) > 0 { 274 key = container.Conditions[0].Params[0].Value 275 conditionParam = container.Conditions[0].Params[0].Name 276 } 277 p := map[string]interface{}{} 278 if key != "" { 279 root[key] = p 280 //add default for condition parameter if available 281 for _, inputParam := range m.Spec.Inputs.Parameters { 282 if inputParam.Name == conditionParam { 283 root[conditionParam] = inputParam.Default 284 } 285 } 286 } else { 287 p = root 288 } 289 if len(container.Command) > 0 { 290 p["containerCommand"] = container.Command[0] 291 } 292 293 putStringIfNotEmpty(p, "containerName", container.Name) 294 putStringIfNotEmpty(p, "containerShell", container.Shell) 295 container.commonConfiguration("docker", &p) 296 297 // Ready command not relevant for main runtime container so far 298 //putStringIfNotEmpty(p, ..., container.ReadyCommand) 299 } 300 301 } 302 303 if len(m.Spec.Sidecars) > 0 { 304 if len(m.Spec.Sidecars[0].Command) > 0 { 305 root["sidecarCommand"] = m.Spec.Sidecars[0].Command[0] 306 } 307 m.Spec.Sidecars[0].commonConfiguration("sidecar", &root) 308 putStringIfNotEmpty(root, "sidecarReadyCommand", m.Spec.Sidecars[0].ReadyCommand) 309 310 // not filled for now since this is not relevant in Kubernetes case 311 //putStringIfNotEmpty(root, "containerPortMappings", m.Spec.Sidecars[0].) 312 } 313 314 if len(m.Spec.Inputs.Resources) > 0 { 315 keys := []string{} 316 resources := map[string][]string{} 317 for _, resource := range m.Spec.Inputs.Resources { 318 if resource.Type == "stash" { 319 key := "" 320 if len(resource.Conditions) > 0 { 321 key = resource.Conditions[0].Params[0].Value 322 } 323 if resources[key] == nil { 324 keys = append(keys, key) 325 resources[key] = []string{} 326 } 327 resources[key] = append(resources[key], resource.Name) 328 } 329 } 330 331 for _, key := range keys { 332 if key == "" { 333 root["stashContent"] = resources[""] 334 } else { 335 if root[key] == nil { 336 root[key] = map[string]interface{}{ 337 "stashContent": resources[key], 338 } 339 } else { 340 p := root[key].(map[string]interface{}) 341 p["stashContent"] = resources[key] 342 } 343 } 344 } 345 } 346 347 c := Config{ 348 Steps: map[string]map[string]interface{}{ 349 stepName: root, 350 }, 351 } 352 353 JSON, err := yaml.Marshal(c) 354 if err != nil { 355 return nil, errors.Wrap(err, "failed to create context defaults") 356 } 357 358 r := ioutil.NopCloser(bytes.NewReader(JSON)) 359 return r, nil 360 } 361 362 // GetResourceParameters retrieves parameters from a named pipeline resource with a defined path 363 func (m *StepData) GetResourceParameters(path, name string) map[string]interface{} { 364 resourceParams := map[string]interface{}{} 365 366 for _, param := range m.Spec.Inputs.Parameters { 367 for _, res := range param.ResourceRef { 368 if res.Name == name { 369 if val := getParameterValue(path, res, param); val != nil { 370 resourceParams[param.Name] = val 371 break 372 } 373 } 374 } 375 } 376 377 return resourceParams 378 } 379 380 func (container *Container) commonConfiguration(keyPrefix string, config *map[string]interface{}) { 381 putMapIfNotEmpty(*config, keyPrefix+"EnvVars", EnvVarsAsMap(container.EnvVars)) 382 putStringIfNotEmpty(*config, keyPrefix+"Image", container.Image) 383 putStringIfNotEmpty(*config, keyPrefix+"Name", container.Name) 384 if container.ImagePullPolicy != "" { 385 (*config)[keyPrefix+"PullImage"] = container.ImagePullPolicy != "Never" 386 } 387 putStringIfNotEmpty(*config, keyPrefix+"Workspace", container.WorkingDir) 388 putSliceIfNotEmpty(*config, keyPrefix+"Options", OptionsAsStringSlice(container.Options)) 389 //putSliceIfNotEmpty(*config, keyPrefix+"VolumeBind", volumeMountsAsStringSlice(container.VolumeMounts)) 390 391 } 392 393 func getParameterValue(path string, res ResourceReference, param StepParameters) interface{} { 394 paramName := res.Param 395 if param.Type != "string" { 396 paramName += ".json" 397 } 398 if val := piperenv.GetResourceParameter(path, res.Name, paramName); len(val) > 0 { 399 if param.Type != "string" { 400 var unmarshalledValue interface{} 401 err := json.Unmarshal([]byte(val), &unmarshalledValue) 402 if err != nil { 403 log.Entry().Debugf("Failed to unmarshal: %v", val) 404 } 405 return unmarshalledValue 406 } 407 return val 408 } 409 return nil 410 } 411 412 // GetReference returns the ResourceReference of the given type 413 func (m *StepParameters) GetReference(refType string) *ResourceReference { 414 for _, ref := range m.ResourceRef { 415 if refType == ref.Type { 416 return &ref 417 } 418 } 419 return nil 420 } 421 422 func getFilterForResourceReferences(params []StepParameters) []string { 423 var filter []string 424 for _, param := range params { 425 reference := param.GetReference("vaultSecret") 426 if reference == nil { 427 reference = param.GetReference("vaultSecretFile") 428 } 429 if reference == nil { 430 return filter 431 } 432 if reference.Name != "" { 433 filter = append(filter, reference.Name) 434 } 435 } 436 return filter 437 } 438 439 // HasReference checks whether StepData contains a parameter that has Reference with the given type 440 func (m *StepData) HasReference(refType string) bool { 441 for _, param := range m.Spec.Inputs.Parameters { 442 if param.GetReference(refType) != nil { 443 return true 444 } 445 } 446 return false 447 } 448 449 // EnvVarsAsMap converts container EnvVars into a map as required by dockerExecute 450 func EnvVarsAsMap(envVars []EnvVar) map[string]string { 451 e := map[string]string{} 452 for _, v := range envVars { 453 e[v.Name] = v.Value 454 } 455 return e 456 } 457 458 // OptionsAsStringSlice converts container options into a string slice as required by dockerExecute 459 func OptionsAsStringSlice(options []Option) []string { 460 e := []string{} 461 for _, v := range options { 462 if len(v.Value) != 0 { 463 e = append(e, fmt.Sprintf("%v %v", v.Name, v.Value)) 464 } else { 465 e = append(e, fmt.Sprintf("%v=", v.Name)) 466 } 467 468 } 469 return e 470 } 471 472 func putStringIfNotEmpty(config map[string]interface{}, key, value string) { 473 if value != "" { 474 config[key] = value 475 } 476 } 477 478 func putMapIfNotEmpty(config map[string]interface{}, key string, value map[string]string) { 479 if len(value) > 0 { 480 config[key] = value 481 } 482 } 483 484 func putSliceIfNotEmpty(config map[string]interface{}, key string, value []string) { 485 if len(value) > 0 { 486 config[key] = value 487 } 488 } 489 490 func ResolveMetadata(gitHubTokens map[string]string, metaDataResolver func() map[string]StepData, stepMetadata string, stepName string) (StepData, error) { 491 492 var metadata StepData 493 494 if stepMetadata != "" { 495 metadataFile, err := OpenPiperFile(stepMetadata, gitHubTokens) 496 if err != nil { 497 return metadata, errors.Wrap(err, "open failed") 498 } 499 500 err = metadata.ReadPipelineStepData(metadataFile) 501 if err != nil { 502 return metadata, errors.Wrap(err, "read failed") 503 } 504 } else { 505 if stepName != "" { 506 if metaDataResolver == nil { 507 return metadata, errors.New("metaDataResolver is nil") 508 } 509 metadataMap := metaDataResolver() 510 var ok bool 511 metadata, ok = metadataMap[stepName] 512 if !ok { 513 return metadata, errors.Errorf("could not retrieve by stepName %v", stepName) 514 } 515 } else { 516 return metadata, errors.Errorf("either one of stepMetadata or stepName parameter has to be passed") 517 } 518 } 519 return metadata, nil 520 } 521 522 //ToDo: Enable this when the Volumes part is also implemented 523 //func volumeMountsAsStringSlice(volumeMounts []VolumeMount) []string { 524 // e := []string{} 525 // for _, v := range volumeMounts { 526 // e = append(e, fmt.Sprintf("%v:%v", v.Name, v.MountPath)) 527 // } 528 // return e 529 //}