github.com/jfrog/jfrog-client-go@v1.40.2/utils/io/fileutils/files.go (about) 1 package fileutils 2 3 import ( 4 "bufio" 5 "bytes" 6 "encoding/json" 7 "errors" 8 "io" 9 "net/url" 10 "os" 11 "os/user" 12 "path/filepath" 13 "reflect" 14 "strings" 15 16 "github.com/jfrog/build-info-go/entities" 17 biutils "github.com/jfrog/build-info-go/utils" 18 gofrog "github.com/jfrog/gofrog/io" 19 "github.com/jfrog/jfrog-client-go/utils/errorutils" 20 ) 21 22 const ( 23 SymlinkFileContent = "" 24 File ItemType = "file" 25 Dir ItemType = "dir" 26 Any ItemType = "any" 27 ) 28 29 func GetFileSeparator() string { 30 return string(os.PathSeparator) 31 } 32 33 // Check if path exists. 34 // If path points at a symlink and `preserveSymLink == true`, 35 // function will return `true` regardless of the symlink target 36 func IsPathExists(path string, preserveSymLink bool) bool { 37 _, err := GetFileInfo(path, preserveSymLink) 38 return !os.IsNotExist(err) 39 } 40 41 // Check if path points at a file. 42 // If path points at a symlink and `preserveSymLink == true`, 43 // function will return `true` regardless of the symlink target 44 func IsFileExists(path string, preserveSymLink bool) (bool, error) { 45 fileInfo, err := GetFileInfo(path, preserveSymLink) 46 if err != nil { 47 if os.IsNotExist(err) { // If doesn't exist, don't omit an error 48 return false, nil 49 } 50 return false, errorutils.CheckError(err) 51 } 52 return !fileInfo.IsDir(), nil 53 } 54 55 // Check if path points at a directory. 56 // If path points at a symlink and `preserveSymLink == true`, 57 // function will return `false` regardless of the symlink target 58 func IsDirExists(path string, preserveSymLink bool) (bool, error) { 59 fileInfo, err := GetFileInfo(path, preserveSymLink) 60 if err != nil { 61 if os.IsNotExist(err) { // If doesn't exist, don't omit an error 62 return false, nil 63 } 64 return false, errorutils.CheckError(err) 65 } 66 return fileInfo.IsDir(), nil 67 } 68 69 // Get the file info of the file in path. 70 // If path points at a symlink and `preserveSymLink == true`, return the file info of the symlink instead 71 func GetFileInfo(path string, preserveSymLink bool) (fileInfo os.FileInfo, err error) { 72 if preserveSymLink { 73 fileInfo, err = os.Lstat(path) 74 } else { 75 fileInfo, err = os.Stat(path) 76 } 77 // We should not do CheckError here, because the error is checked by the calling functions. 78 return 79 } 80 81 func IsDirEmpty(path string) (isEmpty bool, err error) { 82 dir, err := os.Open(path) 83 if errorutils.CheckError(err) != nil { 84 return 85 } 86 defer func() { 87 err = errors.Join(err, errorutils.CheckError(dir.Close())) 88 }() 89 90 _, err = dir.Readdirnames(1) 91 if err == io.EOF { 92 isEmpty = true 93 err = nil 94 return 95 } 96 err = errorutils.CheckError(err) 97 return 98 } 99 100 func IsPathSymlink(path string) bool { 101 f, _ := os.Lstat(path) 102 return f != nil && IsFileSymlink(f) 103 } 104 105 func IsFileSymlink(file os.FileInfo) bool { 106 return file.Mode()&os.ModeSymlink != 0 107 } 108 109 // Return the file's name and dir of a given path by finding the index of the last separator in the path. 110 // Support separators : "/" , "\\" and "\\\\" 111 func GetFileAndDirFromPath(path string) (fileName, dir string) { 112 index1 := strings.LastIndex(path, "/") 113 index2 := strings.LastIndex(path, "\\") 114 var index int 115 offset := 0 116 if index1 >= index2 { 117 index = index1 118 } else { 119 index = index2 120 // Check if the last separator is "\\\\" or "\\". 121 index3 := strings.LastIndex(path, "\\\\") 122 if index3 != -1 && index2-index3 == 1 { 123 offset = 1 124 } 125 } 126 if index != -1 { 127 fileName = path[index+1:] 128 // If the last separator is "\\\\" index will contain the index of the last "\\" , 129 // to get the dir path (without separator suffix) we will use the offset's value. 130 dir = path[:index-offset] 131 return 132 } 133 fileName = path 134 dir = "" 135 return 136 } 137 138 // Get the local path and filename from original file name and path according to targetPath 139 func GetLocalPathAndFile(originalFileName, relativePath, targetPath string, flat bool, placeholdersUsed bool) (localTargetPath, fileName string) { 140 targetFileName, targetDirPath := GetFileAndDirFromPath(targetPath) 141 // Remove double slashes and double backslashes that may appear in the path 142 localTargetPath = filepath.Clean(targetDirPath) 143 // When placeholders are used, the file path shouldn't be taken into account (or in other words, flat = true). 144 if !flat && !placeholdersUsed { 145 localTargetPath = filepath.Join(targetDirPath, relativePath) 146 } 147 148 fileName = originalFileName 149 // '.' as a target path is equivalent to an empty target path. 150 if targetFileName != "" && targetFileName != "." { 151 fileName = targetFileName 152 } 153 return 154 } 155 156 // Return the recursive list of files and directories in the specified path 157 func ListFilesRecursiveWalkIntoDirSymlink(path string, walkIntoDirSymlink bool) (fileList []string, err error) { 158 fileList = []string{} 159 err = gofrog.Walk(path, func(path string, f os.FileInfo, err error) error { 160 fileList = append(fileList, path) 161 return nil 162 }, walkIntoDirSymlink) 163 err = errorutils.CheckError(err) 164 return 165 } 166 167 // Return all files in the specified path who satisfy the filter func. Not recursive. 168 func ListFilesByFilterFunc(path string, filterFunc func(filePath string) (bool, error)) ([]string, error) { 169 sep := GetFileSeparator() 170 if !strings.HasSuffix(path, sep) { 171 path += sep 172 } 173 var fileList []string 174 files, _ := os.ReadDir(path) 175 path = strings.TrimPrefix(path, "."+sep) 176 177 for _, f := range files { 178 filePath := path + f.Name() 179 satisfy, err := filterFunc(filePath) 180 if err != nil { 181 return nil, err 182 } 183 if !satisfy { 184 continue 185 } 186 exists, err := IsFileExists(filePath, false) 187 if err != nil { 188 return nil, err 189 } 190 if exists { 191 fileList = append(fileList, filePath) 192 continue 193 } 194 195 // Checks if the filepath is a symlink. 196 if IsPathSymlink(filePath) { 197 // Gets the file info of the symlink. 198 file, err := GetFileInfo(filePath, false) 199 if errorutils.CheckError(err) != nil { 200 return nil, err 201 } 202 // Checks if the symlink is a file. 203 if !file.IsDir() { 204 fileList = append(fileList, filePath) 205 } 206 } 207 } 208 return fileList, nil 209 } 210 211 // Return the list of files and directories in the specified path 212 func ListFiles(path string, includeDirs bool) ([]string, error) { 213 sep := GetFileSeparator() 214 if !strings.HasSuffix(path, sep) { 215 path += sep 216 } 217 fileList := []string{} 218 files, _ := os.ReadDir(path) 219 path = strings.TrimPrefix(path, "."+sep) 220 221 for _, f := range files { 222 filePath := path + f.Name() 223 exists, err := IsFileExists(filePath, false) 224 if err != nil { 225 return nil, err 226 } 227 if exists || IsPathSymlink(filePath) { 228 fileList = append(fileList, filePath) 229 } else if includeDirs { 230 isDir, err := IsDirExists(filePath, false) 231 if err != nil { 232 return nil, err 233 } 234 if isDir { 235 fileList = append(fileList, filePath) 236 } 237 } 238 } 239 return fileList, nil 240 } 241 242 func GetUploadRequestContent(file *os.File) io.Reader { 243 if file == nil { 244 return bytes.NewBuffer([]byte(SymlinkFileContent)) 245 } 246 return bufio.NewReader(file) 247 } 248 249 func GetFileSize(file *os.File) (int64, error) { 250 size := int64(0) 251 if file != nil { 252 fileInfo, err := file.Stat() 253 if errorutils.CheckError(err) != nil { 254 return size, err 255 } 256 size = fileInfo.Size() 257 } 258 return size, nil 259 } 260 261 func CreateFilePath(localPath, fileName string) (string, error) { 262 if localPath != "" { 263 err := os.MkdirAll(localPath, 0777) 264 if errorutils.CheckError(err) != nil { 265 return "", err 266 } 267 fileName = filepath.Join(localPath, fileName) 268 } 269 return fileName, nil 270 } 271 272 func CreateDirIfNotExist(path string) error { 273 exist, err := IsDirExists(path, false) 274 if exist || err != nil { 275 return err 276 } 277 _, err = CreateFilePath(path, "") 278 return err 279 } 280 281 // Reads the content of the file in the source path and appends it to 282 // the file in the destination path. 283 func AppendFile(srcPath string, destFile *os.File) (err error) { 284 srcFile, err := os.Open(srcPath) 285 if errorutils.CheckError(err) != nil { 286 return 287 } 288 289 defer func() { 290 err = errors.Join(err, errorutils.CheckError(srcFile.Close())) 291 }() 292 293 reader := bufio.NewReader(srcFile) 294 295 writer := bufio.NewWriter(destFile) 296 buf := make([]byte, 1024000) 297 for { 298 var n int 299 n, err = reader.Read(buf) 300 if err != io.EOF { 301 err = errorutils.CheckError(err) 302 if err != nil { 303 return err 304 } 305 } 306 if n == 0 { 307 break 308 } 309 _, err = writer.Write(buf[:n]) 310 err = errorutils.CheckError(err) 311 if err != nil { 312 return err 313 } 314 } 315 err = writer.Flush() 316 return errorutils.CheckError(err) 317 } 318 319 func GetHomeDir() string { 320 home := os.Getenv("HOME") 321 if home != "" { 322 return home 323 } 324 home = os.Getenv("USERPROFILE") 325 if home != "" { 326 return home 327 } 328 currentUser, err := user.Current() 329 if err == nil { 330 return currentUser.HomeDir 331 } 332 return "" 333 } 334 335 func IsSshUrl(urlPath string) bool { 336 u, err := url.Parse(urlPath) 337 if err != nil { 338 return false 339 } 340 return strings.ToLower(u.Scheme) == "ssh" 341 } 342 343 func ReadFile(filePath string) ([]byte, error) { 344 content, err := os.ReadFile(filePath) 345 return content, errorutils.CheckError(err) 346 } 347 348 func GetFileDetails(filePath string, includeChecksums bool) (details *FileDetails, err error) { 349 details = new(FileDetails) 350 if includeChecksums { 351 details.Checksum, err = calcChecksumDetails(filePath) 352 if err != nil { 353 return details, err 354 } 355 } else { 356 details.Checksum = entities.Checksum{} 357 } 358 359 file, err := os.Open(filePath) 360 defer func() { 361 err = errors.Join(err, errorutils.CheckError(file.Close())) 362 }() 363 if err != nil { 364 return nil, errorutils.CheckError(err) 365 } 366 fileInfo, err := file.Stat() 367 if err != nil { 368 return nil, errorutils.CheckError(err) 369 } 370 details.Size = fileInfo.Size() 371 return details, nil 372 } 373 374 func calcChecksumDetails(filePath string) (checksum entities.Checksum, err error) { 375 file, err := os.Open(filePath) 376 defer func() { 377 err = errors.Join(err, errorutils.CheckError(file.Close())) 378 }() 379 if err != nil { 380 return entities.Checksum{}, errorutils.CheckError(err) 381 } 382 return calcChecksumDetailsFromReader(file) 383 } 384 385 func GetFileDetailsFromReader(reader io.Reader, includeChecksums bool) (details *FileDetails, err error) { 386 details = new(FileDetails) 387 388 pr, pw := io.Pipe() 389 defer func() { 390 err = errors.Join(err, errorutils.CheckError(pr.Close())) 391 }() 392 393 go func() { 394 defer func() { 395 err = errors.Join(err, errorutils.CheckError(pw.Close())) 396 }() 397 details.Size, err = io.Copy(pw, reader) 398 }() 399 400 if includeChecksums { 401 details.Checksum, err = calcChecksumDetailsFromReader(pr) 402 } 403 return 404 } 405 406 func calcChecksumDetailsFromReader(reader io.Reader) (entities.Checksum, error) { 407 checksums, err := biutils.CalcChecksums(reader) 408 if err != nil { 409 return entities.Checksum{}, errorutils.CheckError(err) 410 } 411 return entities.Checksum{Md5: checksums[biutils.MD5], Sha1: checksums[biutils.SHA1], Sha256: checksums[biutils.SHA256]}, nil 412 } 413 414 type FileDetails struct { 415 Checksum entities.Checksum 416 Size int64 417 } 418 419 // Removing the provided path from the filesystem 420 func RemovePath(testPath string) error { 421 if file, err := os.Stat(testPath); err == nil { 422 if file.IsDir() { 423 err = RemoveTempDir(testPath) 424 } else { 425 err = errorutils.CheckError(os.Remove(testPath)) 426 } 427 // Delete the path 428 if err != nil { 429 return errors.New("Cannot remove path: " + testPath + " due to: " + err.Error()) 430 } 431 } 432 return nil 433 } 434 435 // Renaming from old path to new path. 436 func RenamePath(oldPath, newPath string) error { 437 err := biutils.CopyDir(oldPath, newPath, true, nil) 438 if err != nil { 439 return errors.New("Error copying directory: " + oldPath + "to" + newPath + err.Error()) 440 } 441 return RemovePath(oldPath) 442 } 443 444 // Returns the path to the directory in which itemToFind is located. 445 // Traversing through directories from current work-dir to root. 446 // itemType determines whether looking for a file or dir. 447 func FindUpstream(itemToFInd string, itemType ItemType) (wd string, exists bool, err error) { 448 // Create a map to store all paths visited, to avoid running in circles. 449 visitedPaths := make(map[string]bool) 450 // Get the current directory. 451 wd, err = os.Getwd() 452 if err != nil { 453 return 454 } 455 origWd := wd 456 defer func() { 457 err = errors.Join(err, errorutils.CheckError(os.Chdir(origWd))) 458 }() 459 // Get the OS root. 460 osRoot := os.Getenv("SYSTEMDRIVE") 461 if osRoot != "" { 462 // If this is a Windows machine: 463 osRoot += "\\" 464 } else { 465 // Unix: 466 osRoot = "/" 467 } 468 469 // Check if the current directory includes itemToFind. If not, check the parent directory 470 // and so on. 471 exists = false 472 for { 473 // If itemToFind is found in the current directory, return the path. 474 switch itemType { 475 case Any: 476 exists = IsPathExists(filepath.Join(wd, itemToFInd), false) 477 case File: 478 exists, err = IsFileExists(filepath.Join(wd, itemToFInd), false) 479 case Dir: 480 exists, err = IsDirExists(filepath.Join(wd, itemToFInd), false) 481 } 482 if err != nil || exists { 483 return 484 } 485 486 // If this the OS root, we can stop. 487 if wd == osRoot { 488 break 489 } 490 491 // Save this path. 492 visitedPaths[wd] = true 493 // CD to the parent directory. 494 wd = filepath.Dir(wd) 495 err = os.Chdir(wd) 496 if err != nil { 497 return "", false, err 498 } 499 500 // If we already visited this directory, it means that there's a loop and we can stop. 501 if visitedPaths[wd] { 502 return "", false, nil 503 } 504 } 505 506 return "", false, nil 507 } 508 509 type ItemType string 510 511 // Returns true if the two files have the same MD5 checksum. 512 func FilesIdentical(file1 string, file2 string) (bool, error) { 513 srcDetails, err := GetFileDetails(file1, true) 514 if err != nil { 515 return false, err 516 } 517 toCompareDetails, err := GetFileDetails(file2, true) 518 if err != nil { 519 return false, err 520 } 521 return srcDetails.Checksum.Md5 == toCompareDetails.Checksum.Md5, nil 522 } 523 524 // JSONEqual compares the JSON from two files. 525 func JsonEqual(filePath1, filePath2 string) (isEqual bool, err error) { 526 reader1, err := os.Open(filePath1) 527 if err != nil { 528 return false, err 529 } 530 defer func() { 531 err = errors.Join(err, errorutils.CheckError(reader1.Close())) 532 }() 533 reader2, err := os.Open(filePath2) 534 if err != nil { 535 return false, err 536 } 537 defer func() { 538 err = errors.Join(err, errorutils.CheckError(reader2.Close())) 539 }() 540 var j, j2 interface{} 541 d := json.NewDecoder(reader1) 542 if err := d.Decode(&j); err != nil { 543 return false, err 544 } 545 d = json.NewDecoder(reader2) 546 if err := d.Decode(&j2); err != nil { 547 return false, err 548 } 549 return reflect.DeepEqual(j2, j), nil 550 } 551 552 // Compares provided Md5 and Sha1 to those of a local file. 553 func IsEqualToLocalFile(localFilePath, md5, sha1 string) (bool, error) { 554 exists, err := IsFileExists(localFilePath, false) 555 if err != nil { 556 return false, err 557 } 558 if !exists { 559 return false, nil 560 } 561 localFileDetails, err := GetFileDetails(localFilePath, true) 562 if err != nil { 563 return false, err 564 } 565 return localFileDetails.Checksum.Md5 == md5 && localFileDetails.Checksum.Sha1 == sha1, nil 566 } 567 568 // Move directory content from one path to another. 569 func MoveDir(fromPath, toPath string) error { 570 err := CreateDirIfNotExist(toPath) 571 if err != nil { 572 return err 573 } 574 575 files, err := ListFiles(fromPath, true) 576 if err != nil { 577 return err 578 } 579 580 for _, v := range files { 581 dir, err := IsDirExists(v, true) 582 if err != nil { 583 return err 584 } 585 586 if dir { 587 toPath := toPath + GetFileSeparator() + filepath.Base(v) 588 err := MoveDir(v, toPath) 589 if err != nil { 590 return err 591 } 592 continue 593 } 594 err = MoveFile(v, filepath.Join(toPath, filepath.Base(v))) 595 if err != nil { 596 return err 597 } 598 } 599 return err 600 } 601 602 // GoLang: os.Rename() give error "invalid cross-device link" for Docker container with Volumes. 603 // MoveFile(source, destination) will work moving file between folders 604 // Therefore, we are using our own implementation (MoveFile) in order to rename files. 605 func MoveFile(sourcePath, destPath string) (err error) { 606 inputFileOpen := true 607 var inputFile *os.File 608 inputFile, err = os.Open(sourcePath) 609 if err != nil { 610 return errorutils.CheckError(err) 611 } 612 defer func() { 613 if inputFileOpen { 614 err = errors.Join(err, errorutils.CheckError(inputFile.Close())) 615 } 616 }() 617 inputFileInfo, err := inputFile.Stat() 618 if err != nil { 619 return errorutils.CheckError(err) 620 } 621 622 var outputFile *os.File 623 outputFile, err = os.Create(destPath) 624 if err != nil { 625 return errorutils.CheckError(err) 626 } 627 defer func() { 628 err = errors.Join(err, errorutils.CheckError(outputFile.Close())) 629 }() 630 631 _, err = io.Copy(outputFile, inputFile) 632 if err != nil { 633 return errorutils.CheckError(err) 634 } 635 err = os.Chmod(destPath, inputFileInfo.Mode()) 636 if err != nil { 637 return errorutils.CheckError(err) 638 } 639 640 // The copy was successful, so now delete the original file 641 err = inputFile.Close() 642 if err != nil { 643 return errorutils.CheckError(err) 644 } 645 inputFileOpen = false 646 err = os.Remove(sourcePath) 647 return errorutils.CheckError(err) 648 } 649 650 // RemoveDirContents removes the contents of the directory, without removing the directory itself. 651 // If it encounters an error before removing all the files, it stops and returns that error. 652 func RemoveDirContents(dirPath string) (err error) { 653 d, err := os.Open(dirPath) 654 if err != nil { 655 return errorutils.CheckError(err) 656 } 657 defer func() { 658 err = errors.Join(err, errorutils.CheckError(d.Close())) 659 }() 660 names, err := d.Readdirnames(-1) 661 if err != nil { 662 return errorutils.CheckError(err) 663 } 664 for _, name := range names { 665 err = os.RemoveAll(filepath.Join(dirPath, name)) 666 if err != nil { 667 return errorutils.CheckError(err) 668 } 669 } 670 return nil 671 }