github.com/devtron-labs/ci-runner@v0.0.0-20240518055909-b2672f3349d7/helper/CacheHelper.go (about) 1 /* 2 * Copyright 2020 Devtron Labs 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 * 16 */ 17 18 package helper 19 20 import ( 21 "github.com/devtron-labs/ci-runner/util" 22 blob_storage "github.com/devtron-labs/common-lib/blob-storage" 23 "log" 24 "os" 25 "os/exec" 26 ) 27 28 func GetCache(ciRequest *CommonWorkflowRequest) error { 29 if !ciRequest.BlobStorageConfigured { 30 log.Println("ignoring cache as storage module not configured ... ") //TODO not needed 31 return nil 32 } 33 if ciRequest.IgnoreDockerCachePull || ciRequest.CacheInvalidate { 34 if !ciRequest.IsPvcMounted { 35 log.Println("ignoring cache ... ") 36 } 37 return nil 38 } 39 log.Println("setting build cache ...............") 40 41 //----------download file 42 blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) 43 cloudHelperBaseConfig := ciRequest.GetCloudHelperBaseConfig(util.BlobStorageObjectTypeCache) 44 request := createBlobStorageRequest(cloudHelperBaseConfig, ciRequest.CiCacheFileName, ciRequest.CiCacheFileName) 45 downloadSuccess, bytesSize, err := blobStorageService.Get(request) 46 if bytesSize >= ciRequest.CacheLimit { 47 log.Println(util.DEVTRON, " cache upper limit exceeded, ignoring old cache") 48 downloadSuccess = false 49 } 50 51 // Extract cache 52 if err == nil && downloadSuccess { 53 extractCmd := exec.Command("tar", "-xvzf", ciRequest.CiCacheFileName) 54 extractCmd.Dir = "/" 55 err = extractCmd.Run() 56 if err != nil { 57 log.Fatal(" Could not extract cache blob ", err) 58 } 59 } else if err != nil { 60 log.Println(util.DEVTRON, "build cache error", err.Error()) 61 } 62 return nil 63 } 64 65 func SyncCache(ciRequest *CommonWorkflowRequest) error { 66 if !ciRequest.BlobStorageConfigured { 67 log.Println("ignoring cache as storage module not configured... ") 68 return nil 69 } 70 if ciRequest.IgnoreDockerCachePush { 71 if ciRequest.IsPvcMounted { 72 return nil 73 } 74 log.Println("ignoring cache as cache push is disabled... ") 75 return nil 76 } 77 err := os.Chdir("/") 78 if err != nil { 79 log.Println(err) 80 return err 81 } 82 util.DeleteFile(ciRequest.CiCacheFileName) 83 // Generate new cache 84 log.Println("Generating new cache") 85 var cachePath string 86 ciBuildConfig := ciRequest.CiBuildConfig 87 if (ciBuildConfig.CiBuildType == SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfig.CiBuildType == MANAGED_DOCKERFILE_BUILD_TYPE) && 88 ciBuildConfig.DockerBuildConfig.CheckForBuildX() { 89 cachePath = util.LOCAL_BUILDX_CACHE_LOCATION 90 } else { 91 cachePath = "/var/lib/docker" 92 } 93 94 tarCmd := exec.Command("tar", "-cvzf", ciRequest.CiCacheFileName, cachePath) 95 tarCmd.Dir = "/" 96 err = tarCmd.Run() 97 if err != nil { 98 log.Fatal("Could not compress cache", err) 99 } 100 101 //aws s3 cp cache.tar.gz s3://ci-caching/ 102 //----------upload file 103 104 log.Println(util.DEVTRON, " -----> pushing new cache") 105 cloudHelperBaseConfig := ciRequest.GetCloudHelperBaseConfig(util.BlobStorageObjectTypeCache) 106 blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil) 107 request := createBlobStorageRequest(cloudHelperBaseConfig, ciRequest.CiCacheFileName, ciRequest.CiCacheFileName) 108 err = blobStorageService.PutWithCommand(request) 109 if err != nil { 110 log.Println(util.DEVTRON, " -----> push err", err) 111 } 112 return err 113 }