github.com/devtron-labs/ci-runner@v0.0.0-20240518055909-b2672f3349d7/helper/CacheHelper.go (about)

     1  /*
     2   *  Copyright 2020 Devtron Labs
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   *
    16   */
    17  
    18  package helper
    19  
    20  import (
    21  	"github.com/devtron-labs/ci-runner/util"
    22  	blob_storage "github.com/devtron-labs/common-lib/blob-storage"
    23  	"log"
    24  	"os"
    25  	"os/exec"
    26  )
    27  
    28  func GetCache(ciRequest *CommonWorkflowRequest) error {
    29  	if !ciRequest.BlobStorageConfigured {
    30  		log.Println("ignoring cache as storage module not configured ... ") //TODO not needed
    31  		return nil
    32  	}
    33  	if ciRequest.IgnoreDockerCachePull || ciRequest.CacheInvalidate {
    34  		if !ciRequest.IsPvcMounted {
    35  			log.Println("ignoring cache ... ")
    36  		}
    37  		return nil
    38  	}
    39  	log.Println("setting build cache ...............")
    40  
    41  	//----------download file
    42  	blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil)
    43  	cloudHelperBaseConfig := ciRequest.GetCloudHelperBaseConfig(util.BlobStorageObjectTypeCache)
    44  	request := createBlobStorageRequest(cloudHelperBaseConfig, ciRequest.CiCacheFileName, ciRequest.CiCacheFileName)
    45  	downloadSuccess, bytesSize, err := blobStorageService.Get(request)
    46  	if bytesSize >= ciRequest.CacheLimit {
    47  		log.Println(util.DEVTRON, " cache upper limit exceeded, ignoring old cache")
    48  		downloadSuccess = false
    49  	}
    50  
    51  	// Extract cache
    52  	if err == nil && downloadSuccess {
    53  		extractCmd := exec.Command("tar", "-xvzf", ciRequest.CiCacheFileName)
    54  		extractCmd.Dir = "/"
    55  		err = extractCmd.Run()
    56  		if err != nil {
    57  			log.Fatal(" Could not extract cache blob ", err)
    58  		}
    59  	} else if err != nil {
    60  		log.Println(util.DEVTRON, "build cache error", err.Error())
    61  	}
    62  	return nil
    63  }
    64  
    65  func SyncCache(ciRequest *CommonWorkflowRequest) error {
    66  	if !ciRequest.BlobStorageConfigured {
    67  		log.Println("ignoring cache as storage module not configured... ")
    68  		return nil
    69  	}
    70  	if ciRequest.IgnoreDockerCachePush {
    71  		if ciRequest.IsPvcMounted {
    72  			return nil
    73  		}
    74  		log.Println("ignoring cache as cache push is disabled... ")
    75  		return nil
    76  	}
    77  	err := os.Chdir("/")
    78  	if err != nil {
    79  		log.Println(err)
    80  		return err
    81  	}
    82  	util.DeleteFile(ciRequest.CiCacheFileName)
    83  	// Generate new cache
    84  	log.Println("Generating new cache")
    85  	var cachePath string
    86  	ciBuildConfig := ciRequest.CiBuildConfig
    87  	if (ciBuildConfig.CiBuildType == SELF_DOCKERFILE_BUILD_TYPE || ciBuildConfig.CiBuildType == MANAGED_DOCKERFILE_BUILD_TYPE) &&
    88  		ciBuildConfig.DockerBuildConfig.CheckForBuildX() {
    89  		cachePath = util.LOCAL_BUILDX_CACHE_LOCATION
    90  	} else {
    91  		cachePath = "/var/lib/docker"
    92  	}
    93  
    94  	tarCmd := exec.Command("tar", "-cvzf", ciRequest.CiCacheFileName, cachePath)
    95  	tarCmd.Dir = "/"
    96  	err = tarCmd.Run()
    97  	if err != nil {
    98  		log.Fatal("Could not compress cache", err)
    99  	}
   100  
   101  	//aws s3 cp cache.tar.gz s3://ci-caching/
   102  	//----------upload file
   103  
   104  	log.Println(util.DEVTRON, " -----> pushing new cache")
   105  	cloudHelperBaseConfig := ciRequest.GetCloudHelperBaseConfig(util.BlobStorageObjectTypeCache)
   106  	blobStorageService := blob_storage.NewBlobStorageServiceImpl(nil)
   107  	request := createBlobStorageRequest(cloudHelperBaseConfig, ciRequest.CiCacheFileName, ciRequest.CiCacheFileName)
   108  	err = blobStorageService.PutWithCommand(request)
   109  	if err != nil {
   110  		log.Println(util.DEVTRON, " -----> push err", err)
   111  	}
   112  	return err
   113  }