github.com/KusionStack/kpm@v0.8.4-0.20240326033734-dc72298a30e5/pkg/utils/utils.go (about)

     1  package utils
     2  
     3  import (
     4  	"archive/tar"
     5  	"bufio"
     6  	"compress/gzip"
     7  	"crypto/sha256"
     8  	"encoding/base64"
     9  	goerrors "errors"
    10  	"fmt"
    11  	"io"
    12  	"log"
    13  	"net/url"
    14  	"os"
    15  	"path/filepath"
    16  	"regexp"
    17  	"strings"
    18  
    19  	"github.com/docker/distribution/reference"
    20  	"github.com/moby/term"
    21  
    22  	"kcl-lang.io/kpm/pkg/constants"
    23  	"kcl-lang.io/kpm/pkg/errors"
    24  	"kcl-lang.io/kpm/pkg/reporter"
    25  )
    26  
    27  // HashDir computes the checksum of a directory by concatenating all files and
    28  // hashing them by sha256.
    29  func HashDir(dir string) (string, error) {
    30  	hasher := sha256.New()
    31  
    32  	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
    33  		if err != nil {
    34  			return err
    35  		}
    36  
    37  		if info.IsDir() {
    38  			return nil
    39  		}
    40  
    41  		// files in the ".git "directory will cause the same repository, cloned at different times,
    42  		// has different checksum.
    43  		for _, ignore := range ignores {
    44  			if strings.Contains(path, ignore) {
    45  				return nil
    46  			}
    47  		}
    48  
    49  		f, err := os.Open(path)
    50  		if err != nil {
    51  			return err
    52  		}
    53  		defer f.Close()
    54  
    55  		if _, err := io.Copy(hasher, f); err != nil {
    56  			return err
    57  		}
    58  
    59  		return nil
    60  	})
    61  
    62  	if err != nil {
    63  		return "", err
    64  	}
    65  
    66  	return base64.StdEncoding.EncodeToString(hasher.Sum(nil)), nil
    67  }
    68  
    69  // StoreToFile will store 'data' into toml file under 'filePath'.
    70  func StoreToFile(filePath string, dataStr string) error {
    71  	file, err := os.Create(filePath)
    72  	if err != nil {
    73  		reporter.ExitWithReport("failed to create file: ", filePath, err)
    74  		return err
    75  	}
    76  	defer file.Close()
    77  
    78  	file, err = os.Create(filePath)
    79  
    80  	if err != nil {
    81  		return err
    82  	}
    83  	defer file.Close()
    84  
    85  	if _, err := io.WriteString(file, dataStr); err != nil {
    86  		return err
    87  	}
    88  	return nil
    89  }
    90  
    91  // ParseRepoNameFromGitUrl get the repo name from git url,
    92  // the repo name in 'https://github.com/xxx/kcl1.git' is 'kcl1'.
    93  func ParseRepoNameFromGitUrl(gitUrl string) string {
    94  	name := filepath.Base(gitUrl)
    95  	return name[:len(name)-len(filepath.Ext(name))]
    96  }
    97  
    98  // CreateFileIfNotExist will check whether the file under a certain path 'filePath/fileName' exists,
    99  // and return an error if it exists, and call the method 'storeFunc' to save the file if it does not exist.
   100  func CreateFileIfNotExist(filePath string, storeFunc func() error) error {
   101  	_, err := os.Stat(filePath)
   102  	if os.IsNotExist(err) {
   103  		err := storeFunc()
   104  		if err != nil {
   105  			return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to create '%s'", filePath))
   106  		}
   107  	} else {
   108  		return reporter.NewErrorEvent(reporter.FileExists, err, fmt.Sprintf("'%s' already exists", filePath))
   109  	}
   110  	return nil
   111  }
   112  
   113  // Whether the file exists
   114  func Exists(path string) (bool, error) {
   115  	_, err := os.Stat(path)
   116  	if os.IsNotExist(err) {
   117  		return false, nil
   118  	}
   119  	if err != nil {
   120  		return false, err
   121  	}
   122  
   123  	return true, nil
   124  }
   125  
   126  // todo: Consider using the OCI tarball as the standard tar format.
   127  var ignores = []string{".git", ".tar"}
   128  
   129  func TarDir(srcDir string, tarPath string) error {
   130  
   131  	fw, err := os.Create(tarPath)
   132  	if err != nil {
   133  		log.Fatal(err)
   134  	}
   135  	defer fw.Close()
   136  
   137  	tw := tar.NewWriter(fw)
   138  	defer tw.Close()
   139  
   140  	err = filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
   141  		if err != nil {
   142  			return err
   143  		}
   144  
   145  		for _, ignore := range ignores {
   146  			if strings.Contains(path, ignore) {
   147  				return nil
   148  			}
   149  		}
   150  
   151  		relPath, _ := filepath.Rel(srcDir, path)
   152  		relPath = filepath.ToSlash(relPath)
   153  
   154  		hdr, err := tar.FileInfoHeader(info, "")
   155  		if err != nil {
   156  			return err
   157  		}
   158  		hdr.Name = relPath
   159  
   160  		if err := tw.WriteHeader(hdr); err != nil {
   161  			return err
   162  		}
   163  
   164  		if info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
   165  			return nil
   166  		}
   167  
   168  		fr, err := os.Open(path)
   169  		if err != nil {
   170  			return err
   171  		}
   172  		defer fr.Close()
   173  
   174  		if _, err := io.Copy(tw, fr); err != nil {
   175  			return err
   176  		}
   177  
   178  		return nil
   179  	})
   180  
   181  	return err
   182  }
   183  
   184  // UnTarDir will extract tar from 'tarPath' to 'destDir'.
   185  func UnTarDir(tarPath string, destDir string) error {
   186  	file, err := os.Open(tarPath)
   187  	if err != nil {
   188  		return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath))
   189  	}
   190  	defer file.Close()
   191  
   192  	tarReader := tar.NewReader(file)
   193  
   194  	for {
   195  		header, err := tarReader.Next()
   196  		if err == io.EOF {
   197  			break
   198  		}
   199  		if err != nil {
   200  			return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath))
   201  		}
   202  
   203  		destFilePath := filepath.Join(destDir, header.Name)
   204  		switch header.Typeflag {
   205  		case tar.TypeDir:
   206  			if err := os.MkdirAll(destFilePath, 0755); err != nil {
   207  				return errors.FailedUnTarKclPackage
   208  			}
   209  		case tar.TypeReg:
   210  			err := os.MkdirAll(filepath.Dir(destFilePath), 0755)
   211  			if err != nil {
   212  				return err
   213  			}
   214  			outFile, err := os.Create(destFilePath)
   215  			if err != nil {
   216  				return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath))
   217  			}
   218  			defer outFile.Close()
   219  
   220  			if _, err := io.Copy(outFile, tarReader); err != nil {
   221  				return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath))
   222  			}
   223  		default:
   224  			return errors.UnknownTarFormat
   225  		}
   226  	}
   227  	return nil
   228  }
   229  
   230  func ExtractTarball(tarPath, destDir string) error {
   231  	f, err := os.Open(tarPath)
   232  	if err != nil {
   233  		return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath))
   234  	}
   235  	defer f.Close()
   236  
   237  	zip, _ := gzip.NewReader(f)
   238  	tarReader := tar.NewReader(zip)
   239  
   240  	for {
   241  		header, err := tarReader.Next()
   242  		if err == io.EOF {
   243  			break
   244  		}
   245  		if err != nil {
   246  			return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath))
   247  		}
   248  
   249  		destFilePath := filepath.Join(destDir, header.Name)
   250  		switch header.Typeflag {
   251  		case tar.TypeDir:
   252  			if err := os.MkdirAll(destFilePath, 0755); err != nil {
   253  				return errors.FailedUnTarKclPackage
   254  			}
   255  		case tar.TypeReg:
   256  			err := os.MkdirAll(filepath.Dir(destFilePath), 0755)
   257  			if err != nil {
   258  				return err
   259  			}
   260  			outFile, err := os.Create(destFilePath)
   261  			if err != nil {
   262  				return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath))
   263  			}
   264  			defer outFile.Close()
   265  
   266  			if _, err := io.Copy(outFile, tarReader); err != nil {
   267  				return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath))
   268  			}
   269  		default:
   270  			return errors.UnknownTarFormat
   271  		}
   272  	}
   273  
   274  	return nil
   275  }
   276  
   277  // DirExists will check whether the directory 'path' exists.
   278  func DirExists(path string) bool {
   279  	_, err := os.Stat(path)
   280  	return err == nil
   281  }
   282  
   283  // DefaultKpmHome create the '.kpm' in the user home and return the path of ".kpm".
   284  func CreateSubdirInUserHome(subdir string) (string, error) {
   285  	homeDir, err := os.UserHomeDir()
   286  	if err != nil {
   287  		return "", reporter.NewErrorEvent(reporter.Bug, err, "internal bugs, failed to load user home directory")
   288  	}
   289  
   290  	dirPath := filepath.Join(homeDir, subdir)
   291  	if !DirExists(dirPath) {
   292  		err = os.MkdirAll(dirPath, 0755)
   293  		if err != nil {
   294  			return "", reporter.NewErrorEvent(reporter.Bug, err, "internal bugs, failed to create directory")
   295  		}
   296  	}
   297  
   298  	return dirPath, nil
   299  }
   300  
   301  // CreateSymlink will create symbolic link named 'newName' for 'oldName',
   302  // and if the symbolic link already exists, it will be deleted and recreated.
   303  func CreateSymlink(oldName, newName string) error {
   304  	if DirExists(newName) {
   305  		err := os.Remove(newName)
   306  		if err != nil {
   307  			return err
   308  		}
   309  	}
   310  
   311  	err := os.Symlink(oldName, newName)
   312  	if err != nil {
   313  		return err
   314  	}
   315  	return nil
   316  }
   317  
   318  // Copied/Adapted from https://github.com/helm/helm
   319  func GetUsernamePassword(usernameOpt string, passwordOpt string, passwordFromStdinOpt bool) (string, string, error) {
   320  	var err error
   321  	username := usernameOpt
   322  	password := passwordOpt
   323  
   324  	if password == "" {
   325  		if username == "" {
   326  			username, err = readLine("Username: ", false)
   327  			if err != nil {
   328  				return "", "", err
   329  			}
   330  			username = strings.TrimSpace(username)
   331  		}
   332  		if username == "" {
   333  			password, err = readLine("Token: ", true)
   334  			if err != nil {
   335  				return "", "", err
   336  			} else if password == "" {
   337  				return "", "", goerrors.New("token required")
   338  			}
   339  		} else {
   340  			password, err = readLine("Password: ", true)
   341  			if err != nil {
   342  				return "", "", err
   343  			} else if password == "" {
   344  				return "", "", goerrors.New("password required")
   345  			}
   346  		}
   347  	}
   348  
   349  	return username, password, nil
   350  }
   351  
   352  // Copied/adapted from https://github.com/helm/helm
   353  func readLine(prompt string, silent bool) (string, error) {
   354  	fmt.Print(prompt)
   355  	if silent {
   356  		fd := os.Stdin.Fd()
   357  		state, err := term.SaveState(fd)
   358  		if err != nil {
   359  			return "", err
   360  		}
   361  		err = term.DisableEcho(fd, state)
   362  		if err != nil {
   363  			return "", err
   364  		}
   365  
   366  		defer func() {
   367  			restoreErr := term.RestoreTerminal(fd, state)
   368  			if err == nil {
   369  				err = restoreErr
   370  			}
   371  		}()
   372  	}
   373  
   374  	reader := bufio.NewReader(os.Stdin)
   375  	line, _, err := reader.ReadLine()
   376  	if err != nil {
   377  		return "", err
   378  	}
   379  	if silent {
   380  		fmt.Println()
   381  	}
   382  
   383  	return string(line), nil
   384  }
   385  
   386  // FindKFiles will find all the '.k' files in the 'path' directory.
   387  func FindKFiles(path string) ([]string, error) {
   388  	var files []string
   389  	info, err := os.Stat(path)
   390  	if err != nil {
   391  		return nil, err
   392  	}
   393  	if !info.IsDir() {
   394  		if strings.HasSuffix(path, ".k") {
   395  			files = append(files, path)
   396  		}
   397  		return files, nil
   398  	}
   399  
   400  	entries, err := os.ReadDir(path)
   401  	if err != nil {
   402  		return nil, err
   403  	}
   404  	for _, entry := range entries {
   405  		if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".k") {
   406  			files = append(files, filepath.Join(path, entry.Name()))
   407  		}
   408  	}
   409  	return files, nil
   410  }
   411  
   412  // RmNewline will remove all the '\r\n' and '\n' in the string 's'.
   413  func RmNewline(s string) string {
   414  	return strings.ReplaceAll(strings.ReplaceAll(s, "\r\n", ""), "\n", "")
   415  }
   416  
   417  // JoinPath will join the 'elem' to the 'base' with '/'.
   418  func JoinPath(base, elem string) string {
   419  	base = strings.TrimSuffix(base, "/")
   420  	elem = strings.TrimPrefix(elem, "/")
   421  	return base + "/" + elem
   422  }
   423  
   424  // IsUrl will check whether the string 'str' is a url.
   425  func IsURL(str string) bool {
   426  	u, err := url.Parse(str)
   427  	return err == nil && u.Scheme != "" && u.Host != ""
   428  }
   429  
   430  // IsGitRepoUrl will check whether the string 'str' is a git repo url
   431  func IsGitRepoUrl(str string) bool {
   432  	r := regexp.MustCompile(`((git|ssh|http(s)?)|(git@[\w\.]+))(:(//)?)([\w\.@\:/\-~]+)(\.git)?(/)?`)
   433  	return r.MatchString(str)
   434  }
   435  
   436  // IsRef will check whether the string 'str' is a reference.
   437  func IsRef(str string) bool {
   438  	_, err := reference.ParseNormalizedNamed(str)
   439  	return err == nil
   440  }
   441  
   442  // IsTar will check whether the string 'str' is a tar path.
   443  func IsTar(str string) bool {
   444  	return strings.HasSuffix(str, constants.TarPathSuffix)
   445  }
   446  
   447  // IsKfile will check whether the string 'str' is a k file path.
   448  func IsKfile(str string) bool {
   449  	return strings.HasSuffix(str, constants.KFilePathSuffix)
   450  }
   451  
   452  // CheckPackageSum will check whether the 'checkedSum' is equal
   453  // to the hash of the package under 'localPath'.
   454  func CheckPackageSum(checkedSum, localPath string) bool {
   455  	if checkedSum == "" {
   456  		return false
   457  	}
   458  
   459  	sum, err := HashDir(localPath)
   460  
   461  	if err != nil {
   462  		return false
   463  	}
   464  
   465  	return checkedSum == sum
   466  }
   467  
   468  // AbsTarPath checks whether path 'tarPath' exists and whether path 'tarPath' ends with '.tar'
   469  // And after checking, absTarPath return the abs path for 'tarPath'.
   470  func AbsTarPath(tarPath string) (string, error) {
   471  	absTarPath, err := filepath.Abs(tarPath)
   472  	if err != nil {
   473  		return "", err
   474  	}
   475  
   476  	if filepath.Ext(absTarPath) != ".tar" {
   477  		return "", errors.InvalidKclPacakgeTar
   478  	} else if !DirExists(absTarPath) {
   479  		return "", errors.KclPacakgeTarNotFound
   480  	}
   481  
   482  	return absTarPath, nil
   483  }