github.com/devtron-labs/ci-runner@v0.0.0-20240518055909-b2672f3349d7/helper/DockerHelper.go (about) 1 /* 2 * Copyright 2020 Devtron Labs 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 * 16 */ 17 18 package helper 19 20 import ( 21 "bytes" 22 "encoding/base64" 23 "encoding/json" 24 "errors" 25 "fmt" 26 "github.com/aws/aws-sdk-go/aws" 27 "github.com/aws/aws-sdk-go/aws/credentials" 28 "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" 29 "github.com/aws/aws-sdk-go/aws/session" 30 "github.com/aws/aws-sdk-go/service/ecr" 31 "github.com/caarlos0/env" 32 "io" 33 "io/ioutil" 34 "log" 35 "os" 36 "os/exec" 37 "path" 38 "path/filepath" 39 "strconv" 40 "strings" 41 "syscall" 42 "time" 43 44 "github.com/devtron-labs/ci-runner/util" 45 ) 46 47 const ( 48 DEVTRON_ENV_VAR_PREFIX = "$devtron_env_" 49 BUILD_ARG_FLAG = "--build-arg" 50 ROOT_PATH = "." 51 BUILDX_K8S_DRIVER_NAME = "devtron-buildx-builder" 52 BUILDX_NODE_NAME = "devtron-buildx-node-" 53 ) 54 55 type DockerHelper interface { 56 StartDockerDaemon(commonWorkflowRequest *CommonWorkflowRequest) 57 DockerLogin(dockerCredentials *DockerCredentials) error 58 BuildArtifact(ciRequest *CommonWorkflowRequest) (string, error) 59 StopDocker() error 60 PushArtifact(dest string) error 61 ExtractDigestForBuildx(dest string) (string, error) 62 CleanBuildxK8sDriver(nodes []map[string]string) error 63 GetDestForNatsEvent(commonWorkflowRequest *CommonWorkflowRequest, dest string) (string, error) 64 } 65 66 type DockerHelperImpl struct { 67 DockerCommandEnv []string 68 } 69 70 func NewDockerHelperImpl() *DockerHelperImpl { 71 return &DockerHelperImpl{ 72 DockerCommandEnv: os.Environ(), 73 } 74 } 75 76 func (impl *DockerHelperImpl) GetDestForNatsEvent(commonWorkflowRequest *CommonWorkflowRequest, dest string) (string, error) { 77 return dest, nil 78 } 79 80 func (impl *DockerHelperImpl) StartDockerDaemon(commonWorkflowRequest *CommonWorkflowRequest) { 81 connection := commonWorkflowRequest.DockerConnection 82 dockerRegistryUrl := commonWorkflowRequest.IntermediateDockerRegistryUrl 83 registryUrl, err := util.ParseUrl(dockerRegistryUrl) 84 if err != nil { 85 log.Fatal(err) 86 } 87 host := registryUrl.Host 88 dockerdstart := "" 89 defaultAddressPoolFlag := "" 90 dockerMtuValueFlag := "" 91 if len(commonWorkflowRequest.DefaultAddressPoolBaseCidr) > 0 { 92 if commonWorkflowRequest.DefaultAddressPoolSize <= 0 { 93 commonWorkflowRequest.DefaultAddressPoolSize = 24 94 } 95 defaultAddressPoolFlag = fmt.Sprintf("--default-address-pool base=%s,size=%d", commonWorkflowRequest.DefaultAddressPoolBaseCidr, commonWorkflowRequest.DefaultAddressPoolSize) 96 } 97 if commonWorkflowRequest.CiBuildDockerMtuValue > 0 { 98 dockerMtuValueFlag = fmt.Sprintf("--mtu=%d", commonWorkflowRequest.CiBuildDockerMtuValue) 99 } 100 if connection == util.INSECURE { 101 dockerdstart = fmt.Sprintf("dockerd %s --insecure-registry %s --host=unix:///var/run/docker.sock %s --host=tcp://0.0.0.0:2375 > /usr/local/bin/nohup.out 2>&1 &", defaultAddressPoolFlag, host, dockerMtuValueFlag) 102 util.LogStage("Insecure Registry") 103 } else { 104 if connection == util.SECUREWITHCERT { 105 os.MkdirAll(fmt.Sprintf("/etc/docker/certs.d/%s", host), os.ModePerm) 106 f, err := os.Create(fmt.Sprintf("/etc/docker/certs.d/%s/ca.crt", host)) 107 108 if err != nil { 109 log.Fatal(err) 110 } 111 112 defer f.Close() 113 114 _, err2 := f.WriteString(commonWorkflowRequest.DockerCert) 115 116 if err2 != nil { 117 log.Fatal(err2) 118 } 119 util.LogStage("Secure with Cert") 120 } 121 dockerdstart = fmt.Sprintf("dockerd %s --host=unix:///var/run/docker.sock %s --host=tcp://0.0.0.0:2375 > /usr/local/bin/nohup.out 2>&1 &", defaultAddressPoolFlag, dockerMtuValueFlag) 122 } 123 cmd := impl.GetCommandToExecute(dockerdstart) 124 out, err := cmd.CombinedOutput() 125 if err != nil { 126 log.Println("failed to start docker daemon") 127 log.Fatal(err) 128 } 129 log.Println("docker daemon started ", string(out)) 130 err = impl.waitForDockerDaemon(util.DOCKER_PS_START_WAIT_SECONDS) 131 if err != nil { 132 log.Fatal("failed to start docker demon", err) 133 } 134 } 135 136 const DOCKER_REGISTRY_TYPE_ECR = "ecr" 137 const DOCKER_REGISTRY_TYPE_DOCKERHUB = "docker-hub" 138 const DOCKER_REGISTRY_TYPE_OTHER = "other" 139 const REGISTRY_TYPE_ARTIFACT_REGISTRY = "artifact-registry" 140 const REGISTRY_TYPE_GCR = "gcr" 141 const JSON_KEY_USERNAME = "_json_key" 142 143 type DockerCredentials struct { 144 DockerUsername, DockerPassword, AwsRegion, AccessKey, SecretKey, DockerRegistryURL, DockerRegistryType string 145 } 146 147 type EnvironmentVariables struct { 148 ShowDockerBuildCmdInLogs bool `env:"SHOW_DOCKER_BUILD_ARGS" envDefault:"true"` 149 } 150 151 func (impl *DockerHelperImpl) GetCommandToExecute(cmd string) *exec.Cmd { 152 execCmd := exec.Command("/bin/sh", "-c", cmd) 153 execCmd.Env = append(execCmd.Env, impl.DockerCommandEnv...) 154 return execCmd 155 } 156 157 func (impl *DockerHelperImpl) DockerLogin(dockerCredentials *DockerCredentials) error { 158 username := dockerCredentials.DockerUsername 159 pwd := dockerCredentials.DockerPassword 160 if dockerCredentials.DockerRegistryType == DOCKER_REGISTRY_TYPE_ECR { 161 accessKey, secretKey := dockerCredentials.AccessKey, dockerCredentials.SecretKey 162 //fmt.Printf("accessKey %s, secretKey %s\n", accessKey, secretKey) 163 164 var creds *credentials.Credentials 165 166 if len(dockerCredentials.AccessKey) == 0 || len(dockerCredentials.SecretKey) == 0 { 167 //fmt.Println("empty accessKey or secretKey") 168 sess, err := session.NewSession(&aws.Config{ 169 Region: &dockerCredentials.AwsRegion, 170 }) 171 if err != nil { 172 log.Println(err) 173 return err 174 } 175 creds = ec2rolecreds.NewCredentials(sess) 176 } else { 177 creds = credentials.NewStaticCredentials(accessKey, secretKey, "") 178 } 179 sess, err := session.NewSession(&aws.Config{ 180 Region: &dockerCredentials.AwsRegion, 181 Credentials: creds, 182 }) 183 if err != nil { 184 log.Println(err) 185 return err 186 } 187 svc := ecr.New(sess) 188 input := &ecr.GetAuthorizationTokenInput{} 189 authData, err := svc.GetAuthorizationToken(input) 190 if err != nil { 191 log.Println(err) 192 return err 193 } 194 // decode token 195 token := authData.AuthorizationData[0].AuthorizationToken 196 decodedToken, err := base64.StdEncoding.DecodeString(*token) 197 if err != nil { 198 log.Println(err) 199 return err 200 } 201 credsSlice := strings.Split(string(decodedToken), ":") 202 username = credsSlice[0] 203 pwd = credsSlice[1] 204 205 } else if (dockerCredentials.DockerRegistryType == REGISTRY_TYPE_GCR || dockerCredentials.DockerRegistryType == REGISTRY_TYPE_ARTIFACT_REGISTRY) && username == JSON_KEY_USERNAME { 206 // for gcr and artifact registry password is already saved as string in DB 207 if strings.HasPrefix(pwd, "'") { 208 pwd = pwd[1:] 209 } 210 if strings.HasSuffix(pwd, "'") { 211 pwd = pwd[:len(pwd)-1] 212 } 213 } 214 host := dockerCredentials.DockerRegistryURL 215 dockerLogin := fmt.Sprintf("docker login -u '%s' -p '%s' '%s' ", username, pwd, host) 216 log.Println("Docker login command ", dockerLogin) 217 awsLoginCmd := impl.GetCommandToExecute(dockerLogin) 218 err := util.RunCommand(awsLoginCmd) 219 if err != nil { 220 log.Println(err) 221 return err 222 } 223 log.Println("Docker login successful with username ", username, " on docker registry URL ", dockerCredentials.DockerRegistryURL) 224 return nil 225 } 226 func (impl *DockerHelperImpl) BuildArtifact(ciRequest *CommonWorkflowRequest) (string, error) { 227 err := impl.DockerLogin(&DockerCredentials{ 228 DockerUsername: ciRequest.DockerUsername, 229 DockerPassword: ciRequest.DockerPassword, 230 AwsRegion: ciRequest.AwsRegion, 231 AccessKey: ciRequest.AccessKey, 232 SecretKey: ciRequest.SecretKey, 233 DockerRegistryURL: ciRequest.IntermediateDockerRegistryUrl, 234 DockerRegistryType: ciRequest.DockerRegistryType, 235 }) 236 if err != nil { 237 return "", err 238 } 239 envVars := &EnvironmentVariables{} 240 err = env.Parse(envVars) 241 if err != nil { 242 log.Println("Error while parsing environment variables", err) 243 } 244 if ciRequest.DockerImageTag == "" { 245 ciRequest.DockerImageTag = "latest" 246 } 247 ciBuildConfig := ciRequest.CiBuildConfig 248 // Docker build, tag image and push 249 dockerFileLocationDir := ciRequest.CheckoutPath 250 log.Println(util.DEVTRON, " docker file location: ", dockerFileLocationDir) 251 252 dest, err := BuildDockerImagePath(ciRequest) 253 if err != nil { 254 return "", err 255 } 256 if ciBuildConfig.CiBuildType == SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfig.CiBuildType == MANAGED_DOCKERFILE_BUILD_TYPE { 257 dockerBuild := "docker build " 258 if ciRequest.CacheInvalidate && ciRequest.IsPvcMounted { 259 dockerBuild = dockerBuild + "--no-cache " 260 } 261 dockerBuildConfig := ciBuildConfig.DockerBuildConfig 262 263 isTargetPlatformSet := dockerBuildConfig.TargetPlatform != "" 264 useBuildx := dockerBuildConfig.CheckForBuildX() 265 dockerBuildxBuild := "docker buildx build " 266 if useBuildx { 267 if ciRequest.CacheInvalidate && ciRequest.IsPvcMounted { 268 dockerBuild = dockerBuildxBuild + "--no-cache " 269 } else { 270 dockerBuild = dockerBuildxBuild + " " 271 } 272 if isTargetPlatformSet { 273 dockerBuild += "--platform " + dockerBuildConfig.TargetPlatform + " " 274 } 275 } 276 dockerBuildFlags := make(map[string]string) 277 dockerBuildArgsMap := dockerBuildConfig.Args 278 for k, v := range dockerBuildArgsMap { 279 flagKey := fmt.Sprintf("%s %s", BUILD_ARG_FLAG, k) 280 if strings.HasPrefix(v, DEVTRON_ENV_VAR_PREFIX) { 281 valueFromEnv := os.Getenv(strings.TrimPrefix(v, DEVTRON_ENV_VAR_PREFIX)) 282 dockerBuildFlags[flagKey] = fmt.Sprintf("=\"%s\"", valueFromEnv) 283 } else { 284 dockerBuildFlags[flagKey] = fmt.Sprintf("=%s", v) 285 } 286 } 287 dockerBuildOptionsMap := dockerBuildConfig.DockerBuildOptions 288 for k, v := range dockerBuildOptionsMap { 289 flagKey := "--" + k 290 if strings.HasPrefix(v, DEVTRON_ENV_VAR_PREFIX) { 291 valueFromEnv := os.Getenv(strings.TrimPrefix(v, DEVTRON_ENV_VAR_PREFIX)) 292 dockerBuildFlags[flagKey] = fmt.Sprintf("=%s", valueFromEnv) 293 } else { 294 dockerBuildFlags[flagKey] = fmt.Sprintf("=%s", v) 295 } 296 } 297 for key, value := range dockerBuildFlags { 298 dockerBuild = dockerBuild + " " + key + value 299 } 300 if !ciRequest.EnableBuildContext || dockerBuildConfig.BuildContext == "" { 301 dockerBuildConfig.BuildContext = ROOT_PATH 302 } 303 dockerBuildConfig.BuildContext = path.Join(ROOT_PATH, dockerBuildConfig.BuildContext) 304 305 dockerfilePath := getDockerfilePath(ciBuildConfig, ciRequest.CheckoutPath) 306 307 if useBuildx { 308 err := checkAndCreateDirectory(util.LOCAL_BUILDX_LOCATION) 309 if err != nil { 310 log.Println(util.DEVTRON, " error in creating LOCAL_BUILDX_LOCATION ", util.LOCAL_BUILDX_LOCATION) 311 return "", err 312 } 313 useBuildxK8sDriver, eligibleK8sDriverNodes := dockerBuildConfig.CheckForBuildXK8sDriver() 314 if useBuildxK8sDriver { 315 err = impl.createBuildxBuilderWithK8sDriver(eligibleK8sDriverNodes, ciRequest.PipelineId, ciRequest.WorkflowId) 316 if err != nil { 317 log.Println(util.DEVTRON, " error in creating buildxDriver , err : ", err.Error()) 318 return "", err 319 } 320 } else { 321 err = impl.createBuildxBuilderForMultiArchBuild() 322 if err != nil { 323 return "", err 324 } 325 } 326 327 cacheEnabled := (ciRequest.IsPvcMounted || ciRequest.BlobStorageConfigured) 328 oldCacheBuildxPath, localCachePath := "", "" 329 330 if cacheEnabled { 331 log.Println(" -----> Setting up cache directory for Buildx") 332 oldCacheBuildxPath = util.LOCAL_BUILDX_LOCATION + "/old" 333 localCachePath = util.LOCAL_BUILDX_CACHE_LOCATION 334 err = setupCacheForBuildx(localCachePath, oldCacheBuildxPath) 335 if err != nil { 336 return "", err 337 } 338 oldCacheBuildxPath = oldCacheBuildxPath + "/cache" 339 } 340 341 dockerBuild = getBuildxBuildCommand(cacheEnabled, dockerBuild, oldCacheBuildxPath, localCachePath, dest, dockerBuildConfig, dockerfilePath) 342 } else { 343 dockerBuild = fmt.Sprintf("%s -f %s --network host -t %s %s", dockerBuild, dockerfilePath, ciRequest.DockerRepository, dockerBuildConfig.BuildContext) 344 } 345 if envVars.ShowDockerBuildCmdInLogs { 346 log.Println("Starting docker build : ", dockerBuild) 347 } else { 348 log.Println("Docker build started..") 349 } 350 err = impl.executeCmd(dockerBuild) 351 if err != nil { 352 return "", err 353 } 354 355 if useBuildK8sDriver, eligibleK8sDriverNodes := dockerBuildConfig.CheckForBuildXK8sDriver(); useBuildK8sDriver { 356 err = impl.CleanBuildxK8sDriver(eligibleK8sDriverNodes) 357 if err != nil { 358 log.Println(util.DEVTRON, " error in cleaning buildx K8s driver ", " err: ", err) 359 } 360 } 361 362 if !useBuildx { 363 err = impl.tagDockerBuild(ciRequest.DockerRepository, dest) 364 if err != nil { 365 return "", err 366 } 367 } 368 } else if ciBuildConfig.CiBuildType == BUILDPACK_BUILD_TYPE { 369 buildPackParams := ciRequest.CiBuildConfig.BuildPackConfig 370 projectPath := buildPackParams.ProjectPath 371 if projectPath == "" || !strings.HasPrefix(projectPath, "./") { 372 projectPath = "./" + projectPath 373 } 374 impl.handleLanguageVersion(projectPath, buildPackParams) 375 buildPackCmd := fmt.Sprintf("pack build %s --path %s --builder %s", dest, projectPath, buildPackParams.BuilderId) 376 BuildPackArgsMap := buildPackParams.Args 377 for k, v := range BuildPackArgsMap { 378 buildPackCmd = buildPackCmd + " --env " + k + "=" + v 379 } 380 381 if len(buildPackParams.BuildPacks) > 0 { 382 for _, buildPack := range buildPackParams.BuildPacks { 383 buildPackCmd = buildPackCmd + " --buildpack " + buildPack 384 } 385 } 386 log.Println(" -----> " + buildPackCmd) 387 err = impl.executeCmd(buildPackCmd) 388 if err != nil { 389 return "", err 390 } 391 builderRmCmdString := "docker image rm " + buildPackParams.BuilderId 392 builderRmCmd := impl.GetCommandToExecute(builderRmCmdString) 393 err := builderRmCmd.Run() 394 if err != nil { 395 return "", err 396 } 397 } 398 399 return dest, nil 400 } 401 402 func getDockerfilePath(CiBuildConfig *CiBuildConfigBean, checkoutPath string) string { 403 var dockerFilePath string 404 if CiBuildConfig.CiBuildType == MANAGED_DOCKERFILE_BUILD_TYPE { 405 dockerFilePath = GetSelfManagedDockerfilePath(checkoutPath) 406 } else { 407 dockerFilePath = CiBuildConfig.DockerBuildConfig.DockerfilePath 408 } 409 return dockerFilePath 410 } 411 412 func getBuildxBuildCommand(cacheEnabled bool, dockerBuild, oldCacheBuildxPath, localCachePath, dest string, dockerBuildConfig *DockerBuildConfig, dockerfilePath string) string { 413 dockerBuild = fmt.Sprintf("%s -f %s -t %s --push %s --network host --allow network.host --allow security.insecure", dockerBuild, dockerfilePath, dest, dockerBuildConfig.BuildContext) 414 if cacheEnabled { 415 dockerBuild = fmt.Sprintf("%s --cache-to=type=local,dest=%s,mode=max --cache-from=type=local,src=%s", dockerBuild, localCachePath, oldCacheBuildxPath) 416 } 417 418 provenanceFlag := dockerBuildConfig.GetProvenanceFlag() 419 dockerBuild = fmt.Sprintf("%s %s", dockerBuild, provenanceFlag) 420 manifestLocation := util.LOCAL_BUILDX_LOCATION + "/manifest.json" 421 dockerBuild = fmt.Sprintf("%s --metadata-file %s", dockerBuild, manifestLocation) 422 423 return dockerBuild 424 } 425 426 func (impl *DockerHelperImpl) handleLanguageVersion(projectPath string, buildpackConfig *BuildPackConfig) { 427 fileData, err := os.ReadFile("/buildpack.json") 428 if err != nil { 429 log.Println("error occurred while reading buildpack json", err) 430 return 431 } 432 var buildpackDataArray []*BuildpackVersionConfig 433 err = json.Unmarshal(fileData, &buildpackDataArray) 434 if err != nil { 435 log.Println("error occurred while reading buildpack json", string(fileData)) 436 return 437 } 438 language := buildpackConfig.Language 439 //languageVersion := buildpackConfig.LanguageVersion 440 buildpackEnvArgs := buildpackConfig.Args 441 languageVersion, present := buildpackEnvArgs["DEVTRON_LANG_VERSION"] 442 if !present { 443 return 444 } 445 var matchedBuildpackConfig *BuildpackVersionConfig 446 for _, versionConfig := range buildpackDataArray { 447 builderPrefix := versionConfig.BuilderPrefix 448 configLanguage := versionConfig.Language 449 builderId := buildpackConfig.BuilderId 450 if strings.HasPrefix(builderId, builderPrefix) && strings.ToLower(language) == configLanguage { 451 matchedBuildpackConfig = versionConfig 452 break 453 } 454 } 455 if matchedBuildpackConfig != nil { 456 fileName := matchedBuildpackConfig.FileName 457 finalPath := filepath.Join(projectPath, "./"+fileName) 458 _, err := os.Stat(finalPath) 459 fileNotExists := errors.Is(err, os.ErrNotExist) 460 if fileNotExists { 461 file, err := os.Create(finalPath) 462 if err != nil { 463 fmt.Println("error occurred while creating file at path " + finalPath) 464 return 465 } 466 entryRegex := matchedBuildpackConfig.EntryRegex 467 languageEntry := fmt.Sprintf(entryRegex, languageVersion) 468 _, err = file.WriteString(languageEntry) 469 log.Println(util.DEVTRON, fmt.Sprintf(" file %s created for language %s with version %s", finalPath, language, languageVersion)) 470 } else if matchedBuildpackConfig.FileOverride { 471 log.Println("final Path is ", finalPath) 472 ext := filepath.Ext(finalPath) 473 if ext == ".json" { 474 jqCmd := fmt.Sprintf("jq '.engines.node' %s", finalPath) 475 outputBytes, err := exec.Command("/bin/sh", "-c", jqCmd).Output() 476 if err != nil { 477 log.Println("error occurred while fetching node version", "err", err) 478 return 479 } 480 if strings.TrimSpace(string(outputBytes)) == "null" { 481 tmpJsonFile := "./tmp.json" 482 versionUpdateCmd := fmt.Sprintf("jq '.engines.node = \"%s\"' %s >%s", languageVersion, finalPath, tmpJsonFile) 483 err := impl.executeCmd(versionUpdateCmd) 484 if err != nil { 485 log.Println("error occurred while inserting node version", "err", err) 486 return 487 } 488 fileReplaceCmd := fmt.Sprintf("mv %s %s", tmpJsonFile, finalPath) 489 err = impl.executeCmd(fileReplaceCmd) 490 if err != nil { 491 log.Println("error occurred while executing cmd ", fileReplaceCmd, "err", err) 492 return 493 } 494 } 495 } 496 } else { 497 log.Println("file already exists, so ignoring version override!!", finalPath) 498 } 499 } 500 501 } 502 503 func (impl *DockerHelperImpl) executeCmd(dockerBuild string) error { 504 dockerBuildCMD := impl.GetCommandToExecute(dockerBuild) 505 err := util.RunCommand(dockerBuildCMD) 506 if err != nil { 507 log.Println(err) 508 } 509 return err 510 } 511 512 func (impl *DockerHelperImpl) tagDockerBuild(dockerRepository string, dest string) error { 513 dockerTag := "docker tag " + dockerRepository + ":latest" + " " + dest 514 log.Println(" -----> " + dockerTag) 515 dockerTagCMD := impl.GetCommandToExecute(dockerTag) 516 err := util.RunCommand(dockerTagCMD) 517 if err != nil { 518 log.Println(err) 519 return err 520 } 521 return nil 522 } 523 524 func setupCacheForBuildx(localCachePath string, oldCacheBuildxPath string) error { 525 err := checkAndCreateDirectory(localCachePath) 526 if err != nil { 527 return err 528 } 529 err = checkAndCreateDirectory(oldCacheBuildxPath) 530 if err != nil { 531 return err 532 } 533 copyContent := "cp -R " + localCachePath + " " + oldCacheBuildxPath 534 copyContentCmd := exec.Command("/bin/sh", "-c", copyContent) 535 err = util.RunCommand(copyContentCmd) 536 if err != nil { 537 log.Println(err) 538 return err 539 } 540 541 cleanContent := "rm -rf " + localCachePath + "/*" 542 cleanContentCmd := exec.Command("/bin/sh", "-c", cleanContent) 543 err = util.RunCommand(cleanContentCmd) 544 if err != nil { 545 log.Println(err) 546 return err 547 } 548 return nil 549 } 550 551 func (impl *DockerHelperImpl) createBuildxBuilder() error { 552 multiPlatformCmd := "docker buildx create --use --buildkitd-flags '--allow-insecure-entitlement network.host --allow-insecure-entitlement security.insecure'" 553 log.Println(" -----> " + multiPlatformCmd) 554 dockerBuildCMD := impl.GetCommandToExecute(multiPlatformCmd) 555 err := util.RunCommand(dockerBuildCMD) 556 if err != nil { 557 log.Println(err) 558 return err 559 } 560 return nil 561 } 562 563 func (impl *DockerHelperImpl) installAllSupportedPlatforms() error { 564 multiPlatformCmd := "docker run --privileged --rm quay.io/devtron/binfmt:stable --install all" 565 log.Println(" -----> " + multiPlatformCmd) 566 dockerBuildCMD := impl.GetCommandToExecute(multiPlatformCmd) 567 err := util.RunCommand(dockerBuildCMD) 568 if err != nil { 569 log.Println(err) 570 return err 571 } 572 return nil 573 } 574 575 func checkAndCreateDirectory(localCachePath string) error { 576 makeDirCmd := "mkdir -p " + localCachePath 577 pathCreateCommand := exec.Command("/bin/sh", "-c", makeDirCmd) 578 err := util.RunCommand(pathCreateCommand) 579 if err != nil { 580 log.Println(err) 581 return err 582 } 583 return nil 584 } 585 586 func BuildDockerImagePath(ciRequest *CommonWorkflowRequest) (string, error) { 587 dest := "" 588 if DOCKER_REGISTRY_TYPE_DOCKERHUB == ciRequest.DockerRegistryType { 589 dest = ciRequest.DockerRepository + ":" + ciRequest.DockerImageTag 590 } else { 591 registryUrl := ciRequest.IntermediateDockerRegistryUrl 592 u, err := util.ParseUrl(registryUrl) 593 if err != nil { 594 log.Println("not a valid docker repository url") 595 return "", err 596 } 597 u.Path = path.Join(u.Path, "/", ciRequest.DockerRepository) 598 dockerRegistryURL := u.Host + u.Path 599 dest = dockerRegistryURL + ":" + ciRequest.DockerImageTag 600 } 601 return dest, nil 602 } 603 604 func (impl *DockerHelperImpl) PushArtifact(dest string) error { 605 //awsLogin := "$(aws ecr get-login --no-include-email --region " + ciRequest.AwsRegion + ")" 606 dockerPush := "docker push " + dest 607 log.Println("-----> " + dockerPush) 608 dockerPushCMD := impl.GetCommandToExecute(dockerPush) 609 err := util.RunCommand(dockerPushCMD) 610 if err != nil { 611 log.Println(err) 612 return err 613 } 614 615 //digest := extractDigestUsingPull(dest) 616 //log.Println("Digest -----> ", digest) 617 //return digest, nil 618 return nil 619 } 620 621 func (impl *DockerHelperImpl) ExtractDigestForBuildx(dest string) (string, error) { 622 623 var digest string 624 var err error 625 manifestLocation := util.LOCAL_BUILDX_LOCATION + "/manifest.json" 626 digest, err = readImageDigestFromManifest(manifestLocation) 627 if err != nil { 628 log.Println("error occurred while extracting digest from manifest reason ", err) 629 err = nil // would extract digest using docker pull cmd 630 } 631 if digest == "" { 632 digest, err = impl.ExtractDigestUsingPull(dest) 633 } 634 log.Println("Digest -----> ", digest) 635 636 return digest, err 637 } 638 639 func (impl *DockerHelperImpl) ExtractDigestUsingPull(dest string) (string, error) { 640 dockerPull := "docker pull " + dest 641 dockerPullCmd := impl.GetCommandToExecute(dockerPull) 642 digest, err := runGetDockerImageDigest(dockerPullCmd) 643 if err != nil { 644 log.Println(err) 645 } 646 return digest, err 647 } 648 649 func runGetDockerImageDigest(cmd *exec.Cmd) (string, error) { 650 var stdBuffer bytes.Buffer 651 mw := io.MultiWriter(os.Stdout, &stdBuffer) 652 cmd.Stdout = mw 653 cmd.Stderr = mw 654 if err := cmd.Run(); err != nil { 655 return "", err 656 } 657 output := stdBuffer.String() 658 outArr := strings.Split(output, "\n") 659 var digest string 660 for _, s := range outArr { 661 if strings.HasPrefix(s, "Digest: ") { 662 digest = s[strings.Index(s, "sha256:"):] 663 } 664 665 } 666 return digest, nil 667 } 668 669 func readImageDigestFromManifest(manifestFilePath string) (string, error) { 670 manifestFile, err := ioutil.ReadFile(manifestFilePath) 671 if err != nil { 672 return "", err 673 } 674 var data map[string]interface{} 675 err = json.Unmarshal(manifestFile, &data) 676 if err != nil { 677 return "", err 678 } 679 imageDigest, found := data["containerimage.digest"] 680 if !found { 681 return "", nil 682 } 683 return imageDigest.(string), nil 684 } 685 686 func (impl *DockerHelperImpl) createBuildxBuilderForMultiArchBuild() error { 687 err := impl.installAllSupportedPlatforms() 688 if err != nil { 689 return err 690 } 691 err = impl.createBuildxBuilder() 692 if err != nil { 693 return err 694 } 695 return nil 696 } 697 698 func (impl *DockerHelperImpl) createBuildxBuilderWithK8sDriver(builderNodes []map[string]string, ciPipelineId, ciWorkflowId int) error { 699 700 if len(builderNodes) == 0 { 701 return errors.New("atleast one node is expected for builder with kubernetes driver") 702 } 703 defaultNodeOpts := builderNodes[0] 704 705 buildxCreate := getBuildxK8sDriverCmd(defaultNodeOpts, ciPipelineId, ciWorkflowId) 706 buildxCreate = fmt.Sprintf("%s %s", buildxCreate, "--use") 707 708 err, errBuf := impl.runCmd(buildxCreate) 709 if err != nil { 710 fmt.Println(util.DEVTRON, "buildxCreate : ", buildxCreate, " err : ", err, " error : ", errBuf.String(), "\n ") 711 return err 712 } 713 714 //appending other nodes to the builder,except default node ,since we already added it 715 for i := 1; i < len(builderNodes); i++ { 716 nodeOpts := builderNodes[i] 717 appendNode := getBuildxK8sDriverCmd(nodeOpts, ciPipelineId, ciWorkflowId) 718 appendNode = fmt.Sprintf("%s %s", appendNode, "--append") 719 720 err, errBuf = impl.runCmd(appendNode) 721 if err != nil { 722 fmt.Println(util.DEVTRON, " appendNode : ", appendNode, " err : ", err, " error : ", errBuf.String(), "\n ") 723 return err 724 } 725 } 726 727 return nil 728 } 729 730 func (impl *DockerHelperImpl) CleanBuildxK8sDriver(nodes []map[string]string) error { 731 nodeNames := make([]string, 0) 732 for _, nOptsMp := range nodes { 733 if nodeName, ok := nOptsMp["node"]; ok && nodeName != "" { 734 nodeNames = append(nodeNames, nodeName) 735 } 736 } 737 err, errBuf := impl.leaveNodesFromBuildxK8sDriver(nodeNames) 738 if err != nil { 739 log.Println(util.DEVTRON, " error in deleting nodes created by ci-runner , err : ", errBuf.String()) 740 return err 741 } 742 log.Println(util.DEVTRON, "successfully cleaned up buildx k8s driver") 743 return nil 744 } 745 746 func (impl *DockerHelperImpl) leaveNodesFromBuildxK8sDriver(nodeNames []string) (error, *bytes.Buffer) { 747 var err error 748 var errBuf *bytes.Buffer 749 defer func() { 750 removeCmd := fmt.Sprintf("docker buildx rm %s", BUILDX_K8S_DRIVER_NAME) 751 err, errBuf = impl.runCmd(removeCmd) 752 if err != nil { 753 log.Println(util.DEVTRON, "error in removing docker buildx err : ", errBuf.String()) 754 } 755 }() 756 for _, node := range nodeNames { 757 cmds := fmt.Sprintf("docker buildx create --name=%s --node=%s --leave", BUILDX_K8S_DRIVER_NAME, node) 758 err, errBuf = impl.runCmd(cmds) 759 if err != nil { 760 log.Println(util.DEVTRON, "error in leaving node : ", errBuf.String()) 761 return err, errBuf 762 } 763 } 764 return err, errBuf 765 } 766 767 func (impl *DockerHelperImpl) runCmd(cmd string) (error, *bytes.Buffer) { 768 fmt.Println(util.DEVTRON, " cmd : ", cmd) 769 builderCreateCmd := impl.GetCommandToExecute(cmd) 770 errBuf := &bytes.Buffer{} 771 builderCreateCmd.Stderr = errBuf 772 err := builderCreateCmd.Run() 773 return err, errBuf 774 } 775 776 func getBuildxK8sDriverCmd(driverOpts map[string]string, ciPipelineId, ciWorkflowId int) string { 777 buildxCreate := "docker buildx create --buildkitd-flags '--allow-insecure-entitlement network.host --allow-insecure-entitlement security.insecure' --name=%s --driver=kubernetes --node=%s --bootstrap " 778 nodeName := driverOpts["node"] 779 if nodeName == "" { 780 nodeName = BUILDX_NODE_NAME + fmt.Sprintf("%v-%v-", ciPipelineId, ciWorkflowId) + util.Generate(3) //need this to generate unique name for builder node in same builder. 781 } 782 buildxCreate = fmt.Sprintf(buildxCreate, BUILDX_K8S_DRIVER_NAME, nodeName) 783 platforms := driverOpts["platform"] 784 if platforms != "" { 785 buildxCreate += " --platform=%s " 786 buildxCreate = fmt.Sprintf(buildxCreate, platforms) 787 } 788 if len(driverOpts["driverOptions"]) > 0 { 789 buildxCreate += " '--driver-opt=%s' " 790 buildxCreate = fmt.Sprintf(buildxCreate, driverOpts["driverOptions"]) 791 } 792 return buildxCreate 793 } 794 795 func (impl *DockerHelperImpl) StopDocker() error { 796 cmd := exec.Command("docker", "ps", "-a", "-q") 797 out, err := cmd.Output() 798 if err != nil { 799 return err 800 } 801 if len(out) > 0 { 802 stopCmdS := "docker stop -t 5 $(docker ps -a -q)" 803 log.Println(util.DEVTRON, " -----> stopping docker container") 804 stopCmd := impl.GetCommandToExecute(stopCmdS) 805 err := util.RunCommand(stopCmd) 806 log.Println(util.DEVTRON, " -----> stopped docker container") 807 if err != nil { 808 log.Fatal(err) 809 return err 810 } 811 removeContainerCmds := "docker rm -v -f $(docker ps -a -q)" 812 log.Println(util.DEVTRON, " -----> removing docker container") 813 removeContainerCmd := impl.GetCommandToExecute(removeContainerCmds) 814 err = util.RunCommand(removeContainerCmd) 815 log.Println(util.DEVTRON, " -----> removed docker container") 816 if err != nil { 817 log.Fatal(err) 818 return err 819 } 820 } 821 file := "/var/run/docker.pid" 822 content, err := ioutil.ReadFile(file) 823 if err != nil { 824 log.Fatal(err) 825 return err 826 } 827 828 pid, err := strconv.Atoi(string(content)) 829 if err != nil { 830 return err 831 } 832 proc, err := os.FindProcess(pid) 833 if err != nil { 834 log.Println(err) 835 return err 836 } 837 // Kill the process 838 err = proc.Signal(syscall.SIGTERM) 839 if err != nil { 840 log.Println(err) 841 return err 842 } 843 log.Println(util.DEVTRON, " -----> checking docker status") 844 impl.DockerdUpCheck() //FIXME: this call should be removed 845 //ensureDockerDaemonHasStopped(20) 846 return nil 847 } 848 849 func (impl *DockerHelperImpl) ensureDockerDaemonHasStopped(retryCount int) error { 850 var err error 851 retry := 0 852 for err == nil { 853 time.Sleep(1 * time.Second) 854 err = impl.DockerdUpCheck() 855 retry++ 856 if retry == retryCount { 857 break 858 } 859 } 860 return err 861 } 862 863 func (impl *DockerHelperImpl) waitForDockerDaemon(retryCount int) error { 864 err := impl.DockerdUpCheck() 865 retry := 0 866 for err != nil { 867 if retry == retryCount { 868 break 869 } 870 time.Sleep(1 * time.Second) 871 err = impl.DockerdUpCheck() 872 retry++ 873 } 874 return err 875 } 876 877 func (impl *DockerHelperImpl) DockerdUpCheck() error { 878 dockerCheck := "docker ps" 879 dockerCheckCmd := impl.GetCommandToExecute(dockerCheck) 880 err := dockerCheckCmd.Run() 881 return err 882 } 883 884 func ValidBuildxK8sDriverOptions(ciRequest *CommonWorkflowRequest) (bool, []map[string]string) { 885 valid := ciRequest != nil && ciRequest.CiBuildConfig != nil && ciRequest.CiBuildConfig.DockerBuildConfig != nil 886 if valid { 887 return ciRequest.CiBuildConfig.DockerBuildConfig.CheckForBuildXK8sDriver() 888 } 889 return false, nil 890 } 891 892 func GetSelfManagedDockerfilePath(checkoutPath string) string { 893 return filepath.Join(util.WORKINGDIR, checkoutPath, "./Dockerfile") 894 }