github.com/jfrog/jfrog-cli-core@v1.12.1/artifactory/commands/generic/download.go (about) 1 package generic 2 3 import ( 4 "errors" 5 "github.com/jfrog/jfrog-cli-core/utils/coreutils" 6 "os" 7 "path/filepath" 8 "strconv" 9 "strings" 10 11 "github.com/jfrog/jfrog-cli-core/artifactory/spec" 12 "github.com/jfrog/jfrog-cli-core/artifactory/utils" 13 "github.com/jfrog/jfrog-client-go/artifactory/buildinfo" 14 "github.com/jfrog/jfrog-client-go/artifactory/services" 15 serviceutils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" 16 clientutils "github.com/jfrog/jfrog-client-go/utils" 17 "github.com/jfrog/jfrog-client-go/utils/errorutils" 18 ioUtils "github.com/jfrog/jfrog-client-go/utils/io" 19 "github.com/jfrog/jfrog-client-go/utils/io/content" 20 "github.com/jfrog/jfrog-client-go/utils/io/fileutils" 21 "github.com/jfrog/jfrog-client-go/utils/log" 22 ) 23 24 type DownloadCommand struct { 25 buildConfiguration *utils.BuildConfiguration 26 GenericCommand 27 configuration *utils.DownloadConfiguration 28 progress ioUtils.ProgressMgr 29 } 30 31 func NewDownloadCommand() *DownloadCommand { 32 return &DownloadCommand{GenericCommand: *NewGenericCommand()} 33 } 34 35 func (dc *DownloadCommand) SetBuildConfiguration(buildConfiguration *utils.BuildConfiguration) *DownloadCommand { 36 dc.buildConfiguration = buildConfiguration 37 return dc 38 } 39 40 func (dc *DownloadCommand) Configuration() *utils.DownloadConfiguration { 41 return dc.configuration 42 } 43 44 func (dc *DownloadCommand) SetConfiguration(configuration *utils.DownloadConfiguration) *DownloadCommand { 45 dc.configuration = configuration 46 return dc 47 } 48 49 func (dc *DownloadCommand) SetProgress(progress ioUtils.ProgressMgr) { 50 dc.progress = progress 51 } 52 53 func (dc *DownloadCommand) ShouldPrompt() bool { 54 return !dc.DryRun() && dc.SyncDeletesPath() != "" && !dc.Quiet() 55 } 56 57 func (dc *DownloadCommand) CommandName() string { 58 return "rt_download" 59 } 60 61 func (dc *DownloadCommand) Run() error { 62 return dc.download() 63 } 64 65 func (dc *DownloadCommand) download() error { 66 // Create Service Manager: 67 servicesManager, err := utils.CreateDownloadServiceManager(dc.serverDetails, dc.configuration.Threads, dc.retries, dc.DryRun(), dc.progress) 68 if err != nil { 69 return err 70 } 71 72 // Build Info Collection: 73 isCollectBuildInfo := len(dc.buildConfiguration.BuildName) > 0 && len(dc.buildConfiguration.BuildNumber) > 0 74 if isCollectBuildInfo && !dc.DryRun() { 75 if err = utils.SaveBuildGeneralDetails(dc.buildConfiguration.BuildName, dc.buildConfiguration.BuildNumber, dc.buildConfiguration.Project); err != nil { 76 return err 77 } 78 } 79 80 var errorOccurred = false 81 var downloadParamsArray []services.DownloadParams 82 // Create DownloadParams for all File-Spec groups. 83 for i := 0; i < len(dc.Spec().Files); i++ { 84 downParams, err := getDownloadParams(dc.Spec().Get(i), dc.configuration) 85 if err != nil { 86 errorOccurred = true 87 log.Error(err) 88 continue 89 } 90 downloadParamsArray = append(downloadParamsArray, downParams) 91 } 92 // Perform download. 93 // In case of build-info collection/sync-deletes operation/a detailed summary is required, we use the download service which provides results file reader, 94 // otherwise we use the download service which provides only general counters. 95 var totalDownloaded, totalFailed int 96 var summary *serviceutils.OperationSummary 97 if isCollectBuildInfo || dc.SyncDeletesPath() != "" || dc.DetailedSummary() { 98 summary, err = servicesManager.DownloadFilesWithSummary(downloadParamsArray...) 99 if err != nil { 100 errorOccurred = true 101 log.Error(err) 102 } 103 if summary != nil { 104 defer summary.ArtifactsDetailsReader.Close() 105 // If 'detailed summary' was requested, then the reader should not be closed here. 106 // It will be closed after it will be used to generate the summary. 107 if dc.DetailedSummary() { 108 dc.result.SetReader(summary.TransferDetailsReader) 109 } else { 110 defer summary.TransferDetailsReader.Close() 111 } 112 totalDownloaded = summary.TotalSucceeded 113 totalFailed = summary.TotalFailed 114 } 115 } else { 116 totalDownloaded, totalFailed, err = servicesManager.DownloadFiles(downloadParamsArray...) 117 if err != nil { 118 errorOccurred = true 119 log.Error(err) 120 } 121 } 122 dc.result.SetSuccessCount(totalDownloaded) 123 dc.result.SetFailCount(totalFailed) 124 // Check for errors. 125 if errorOccurred { 126 return errors.New("Download finished with errors, please review the logs.") 127 } 128 if dc.DryRun() { 129 dc.result.SetSuccessCount(totalDownloaded) 130 dc.result.SetFailCount(0) 131 return err 132 } else if dc.SyncDeletesPath() != "" { 133 absSyncDeletesPath, err := filepath.Abs(dc.SyncDeletesPath()) 134 if err != nil { 135 return errorutils.CheckError(err) 136 } 137 if _, err = os.Stat(absSyncDeletesPath); err == nil { 138 // Unmarshal the local paths of the downloaded files from the results file reader 139 tmpRoot, err := createDownloadResultEmptyTmpReflection(summary.TransferDetailsReader) 140 defer fileutils.RemoveTempDir(tmpRoot) 141 if err != nil { 142 return err 143 } 144 walkFn := createSyncDeletesWalkFunction(tmpRoot) 145 err = fileutils.Walk(dc.SyncDeletesPath(), walkFn, false) 146 if err != nil { 147 return errorutils.CheckError(err) 148 } 149 } else if os.IsNotExist(err) { 150 log.Info("Sync-deletes path", absSyncDeletesPath, "does not exists.") 151 } 152 } 153 log.Debug("Downloaded", strconv.Itoa(totalDownloaded), "artifacts.") 154 155 // Build Info 156 if isCollectBuildInfo { 157 var buildDependencies []buildinfo.Dependency 158 buildDependencies, err = serviceutils.ConvertArtifactsDetailsToBuildInfoDependencies(summary.ArtifactsDetailsReader) 159 if err != nil { 160 return err 161 } 162 populateFunc := func(partial *buildinfo.Partial) { 163 partial.Dependencies = buildDependencies 164 partial.ModuleId = dc.buildConfiguration.Module 165 partial.ModuleType = buildinfo.Generic 166 } 167 err = utils.SavePartialBuildInfo(dc.buildConfiguration.BuildName, dc.buildConfiguration.BuildNumber, dc.buildConfiguration.Project, populateFunc) 168 } 169 170 return err 171 } 172 173 func getDownloadParams(f *spec.File, configuration *utils.DownloadConfiguration) (downParams services.DownloadParams, err error) { 174 downParams = services.NewDownloadParams() 175 downParams.ArtifactoryCommonParams, err = f.ToArtifactoryCommonParams() 176 if err != nil { 177 return 178 } 179 downParams.Symlink = configuration.Symlink 180 downParams.MinSplitSize = configuration.MinSplitSize 181 downParams.SplitCount = configuration.SplitCount 182 183 downParams.Recursive, err = f.IsRecursive(true) 184 if err != nil { 185 return 186 } 187 188 downParams.IncludeDirs, err = f.IsIncludeDirs(false) 189 if err != nil { 190 return 191 } 192 193 downParams.Flat, err = f.IsFlat(false) 194 if err != nil { 195 return 196 } 197 198 downParams.Explode, err = f.IsExplode(false) 199 if err != nil { 200 return 201 } 202 203 downParams.ValidateSymlink, err = f.IsVlidateSymlinks(false) 204 if err != nil { 205 return 206 } 207 208 downParams.ExcludeArtifacts, err = f.IsExcludeArtifacts(false) 209 if err != nil { 210 return 211 } 212 213 downParams.IncludeDeps, err = f.IsIncludeDeps(false) 214 if err != nil { 215 return 216 } 217 218 downParams.Transitive = strings.ToLower(os.Getenv(coreutils.TransitiveDownload)) == "true" 219 220 return 221 } 222 223 // We will create the same downloaded hierarchies under a temp directory with 0-size files. 224 // We will use this "empty reflection" of the download operation to determine whether a file was downloaded or not while walking the real filesystem from sync-deletes root. 225 func createDownloadResultEmptyTmpReflection(reader *content.ContentReader) (tmpRoot string, err error) { 226 tmpRoot, err = fileutils.CreateTempDir() 227 if errorutils.CheckError(err) != nil { 228 return 229 } 230 for path := new(clientutils.FileTransferDetails); reader.NextRecord(path) == nil; path = new(clientutils.FileTransferDetails) { 231 var absDownloadPath string 232 absDownloadPath, err = filepath.Abs(path.TargetPath) 233 if errorutils.CheckError(err) != nil { 234 return 235 } 236 legalPath := createLegalPath(tmpRoot, absDownloadPath) 237 tmpFileRoot := filepath.Dir(legalPath) 238 err = os.MkdirAll(tmpFileRoot, os.ModePerm) 239 if errorutils.CheckError(err) != nil { 240 return 241 } 242 var tmpFile *os.File 243 tmpFile, err = os.Create(legalPath) 244 if errorutils.CheckError(err) != nil { 245 return 246 } 247 err = tmpFile.Close() 248 if errorutils.CheckError(err) != nil { 249 return 250 } 251 } 252 return 253 } 254 255 // Creates absolute path for temp file suitable for all environments 256 func createLegalPath(root, path string) string { 257 // Avoid concatenating the volume name (e.g "C://") in Windows environment. 258 volumeName := filepath.VolumeName(path) 259 if volumeName != "" && strings.HasPrefix(path, volumeName) { 260 alternativeVolumeName := "VolumeName" + string(volumeName[0]) 261 path = strings.Replace(path, volumeName, alternativeVolumeName, 1) 262 } 263 // Join the current path to the temp root provided. 264 path = filepath.Join(root, path) 265 return path 266 } 267 268 func createSyncDeletesWalkFunction(tempRoot string) fileutils.WalkFunc { 269 return func(path string, info os.FileInfo, err error) error { 270 // Convert path to absolute path 271 path, err = filepath.Abs(path) 272 if errorutils.CheckError(err) != nil { 273 return err 274 } 275 pathToCheck := createLegalPath(tempRoot, path) 276 277 // If the path exists under the temp root directory, it means it's been downloaded during the last operations, and cannot be deleted. 278 if fileutils.IsPathExists(pathToCheck, false) { 279 return nil 280 } 281 log.Info("Deleting:", path) 282 if info.IsDir() { 283 // If current path is a dir - remove all content and return SkipDir to stop walking this path 284 err = os.RemoveAll(path) 285 if err == nil { 286 return fileutils.SkipDir 287 } 288 } else { 289 // Path is a file 290 err = os.Remove(path) 291 } 292 293 return errorutils.CheckError(err) 294 } 295 }