github.com/KusionStack/kpm@v0.8.4-0.20240326033734-dc72298a30e5/pkg/utils/utils.go (about) 1 package utils 2 3 import ( 4 "archive/tar" 5 "bufio" 6 "compress/gzip" 7 "crypto/sha256" 8 "encoding/base64" 9 goerrors "errors" 10 "fmt" 11 "io" 12 "log" 13 "net/url" 14 "os" 15 "path/filepath" 16 "regexp" 17 "strings" 18 19 "github.com/docker/distribution/reference" 20 "github.com/moby/term" 21 22 "kcl-lang.io/kpm/pkg/constants" 23 "kcl-lang.io/kpm/pkg/errors" 24 "kcl-lang.io/kpm/pkg/reporter" 25 ) 26 27 // HashDir computes the checksum of a directory by concatenating all files and 28 // hashing them by sha256. 29 func HashDir(dir string) (string, error) { 30 hasher := sha256.New() 31 32 err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { 33 if err != nil { 34 return err 35 } 36 37 if info.IsDir() { 38 return nil 39 } 40 41 // files in the ".git "directory will cause the same repository, cloned at different times, 42 // has different checksum. 43 for _, ignore := range ignores { 44 if strings.Contains(path, ignore) { 45 return nil 46 } 47 } 48 49 f, err := os.Open(path) 50 if err != nil { 51 return err 52 } 53 defer f.Close() 54 55 if _, err := io.Copy(hasher, f); err != nil { 56 return err 57 } 58 59 return nil 60 }) 61 62 if err != nil { 63 return "", err 64 } 65 66 return base64.StdEncoding.EncodeToString(hasher.Sum(nil)), nil 67 } 68 69 // StoreToFile will store 'data' into toml file under 'filePath'. 70 func StoreToFile(filePath string, dataStr string) error { 71 file, err := os.Create(filePath) 72 if err != nil { 73 reporter.ExitWithReport("failed to create file: ", filePath, err) 74 return err 75 } 76 defer file.Close() 77 78 file, err = os.Create(filePath) 79 80 if err != nil { 81 return err 82 } 83 defer file.Close() 84 85 if _, err := io.WriteString(file, dataStr); err != nil { 86 return err 87 } 88 return nil 89 } 90 91 // ParseRepoNameFromGitUrl get the repo name from git url, 92 // the repo name in 'https://github.com/xxx/kcl1.git' is 'kcl1'. 93 func ParseRepoNameFromGitUrl(gitUrl string) string { 94 name := filepath.Base(gitUrl) 95 return name[:len(name)-len(filepath.Ext(name))] 96 } 97 98 // CreateFileIfNotExist will check whether the file under a certain path 'filePath/fileName' exists, 99 // and return an error if it exists, and call the method 'storeFunc' to save the file if it does not exist. 100 func CreateFileIfNotExist(filePath string, storeFunc func() error) error { 101 _, err := os.Stat(filePath) 102 if os.IsNotExist(err) { 103 err := storeFunc() 104 if err != nil { 105 return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to create '%s'", filePath)) 106 } 107 } else { 108 return reporter.NewErrorEvent(reporter.FileExists, err, fmt.Sprintf("'%s' already exists", filePath)) 109 } 110 return nil 111 } 112 113 // Whether the file exists 114 func Exists(path string) (bool, error) { 115 _, err := os.Stat(path) 116 if os.IsNotExist(err) { 117 return false, nil 118 } 119 if err != nil { 120 return false, err 121 } 122 123 return true, nil 124 } 125 126 // todo: Consider using the OCI tarball as the standard tar format. 127 var ignores = []string{".git", ".tar"} 128 129 func TarDir(srcDir string, tarPath string) error { 130 131 fw, err := os.Create(tarPath) 132 if err != nil { 133 log.Fatal(err) 134 } 135 defer fw.Close() 136 137 tw := tar.NewWriter(fw) 138 defer tw.Close() 139 140 err = filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { 141 if err != nil { 142 return err 143 } 144 145 for _, ignore := range ignores { 146 if strings.Contains(path, ignore) { 147 return nil 148 } 149 } 150 151 relPath, _ := filepath.Rel(srcDir, path) 152 relPath = filepath.ToSlash(relPath) 153 154 hdr, err := tar.FileInfoHeader(info, "") 155 if err != nil { 156 return err 157 } 158 hdr.Name = relPath 159 160 if err := tw.WriteHeader(hdr); err != nil { 161 return err 162 } 163 164 if info.IsDir() || info.Mode()&os.ModeSymlink != 0 { 165 return nil 166 } 167 168 fr, err := os.Open(path) 169 if err != nil { 170 return err 171 } 172 defer fr.Close() 173 174 if _, err := io.Copy(tw, fr); err != nil { 175 return err 176 } 177 178 return nil 179 }) 180 181 return err 182 } 183 184 // UnTarDir will extract tar from 'tarPath' to 'destDir'. 185 func UnTarDir(tarPath string, destDir string) error { 186 file, err := os.Open(tarPath) 187 if err != nil { 188 return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath)) 189 } 190 defer file.Close() 191 192 tarReader := tar.NewReader(file) 193 194 for { 195 header, err := tarReader.Next() 196 if err == io.EOF { 197 break 198 } 199 if err != nil { 200 return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath)) 201 } 202 203 destFilePath := filepath.Join(destDir, header.Name) 204 switch header.Typeflag { 205 case tar.TypeDir: 206 if err := os.MkdirAll(destFilePath, 0755); err != nil { 207 return errors.FailedUnTarKclPackage 208 } 209 case tar.TypeReg: 210 err := os.MkdirAll(filepath.Dir(destFilePath), 0755) 211 if err != nil { 212 return err 213 } 214 outFile, err := os.Create(destFilePath) 215 if err != nil { 216 return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath)) 217 } 218 defer outFile.Close() 219 220 if _, err := io.Copy(outFile, tarReader); err != nil { 221 return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath)) 222 } 223 default: 224 return errors.UnknownTarFormat 225 } 226 } 227 return nil 228 } 229 230 func ExtractTarball(tarPath, destDir string) error { 231 f, err := os.Open(tarPath) 232 if err != nil { 233 return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath)) 234 } 235 defer f.Close() 236 237 zip, _ := gzip.NewReader(f) 238 tarReader := tar.NewReader(zip) 239 240 for { 241 header, err := tarReader.Next() 242 if err == io.EOF { 243 break 244 } 245 if err != nil { 246 return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath)) 247 } 248 249 destFilePath := filepath.Join(destDir, header.Name) 250 switch header.Typeflag { 251 case tar.TypeDir: 252 if err := os.MkdirAll(destFilePath, 0755); err != nil { 253 return errors.FailedUnTarKclPackage 254 } 255 case tar.TypeReg: 256 err := os.MkdirAll(filepath.Dir(destFilePath), 0755) 257 if err != nil { 258 return err 259 } 260 outFile, err := os.Create(destFilePath) 261 if err != nil { 262 return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath)) 263 } 264 defer outFile.Close() 265 266 if _, err := io.Copy(outFile, tarReader); err != nil { 267 return reporter.NewErrorEvent(reporter.FailedCreateFile, err, fmt.Sprintf("failed to open '%s'", tarPath)) 268 } 269 default: 270 return errors.UnknownTarFormat 271 } 272 } 273 274 return nil 275 } 276 277 // DirExists will check whether the directory 'path' exists. 278 func DirExists(path string) bool { 279 _, err := os.Stat(path) 280 return err == nil 281 } 282 283 // DefaultKpmHome create the '.kpm' in the user home and return the path of ".kpm". 284 func CreateSubdirInUserHome(subdir string) (string, error) { 285 homeDir, err := os.UserHomeDir() 286 if err != nil { 287 return "", reporter.NewErrorEvent(reporter.Bug, err, "internal bugs, failed to load user home directory") 288 } 289 290 dirPath := filepath.Join(homeDir, subdir) 291 if !DirExists(dirPath) { 292 err = os.MkdirAll(dirPath, 0755) 293 if err != nil { 294 return "", reporter.NewErrorEvent(reporter.Bug, err, "internal bugs, failed to create directory") 295 } 296 } 297 298 return dirPath, nil 299 } 300 301 // CreateSymlink will create symbolic link named 'newName' for 'oldName', 302 // and if the symbolic link already exists, it will be deleted and recreated. 303 func CreateSymlink(oldName, newName string) error { 304 if DirExists(newName) { 305 err := os.Remove(newName) 306 if err != nil { 307 return err 308 } 309 } 310 311 err := os.Symlink(oldName, newName) 312 if err != nil { 313 return err 314 } 315 return nil 316 } 317 318 // Copied/Adapted from https://github.com/helm/helm 319 func GetUsernamePassword(usernameOpt string, passwordOpt string, passwordFromStdinOpt bool) (string, string, error) { 320 var err error 321 username := usernameOpt 322 password := passwordOpt 323 324 if password == "" { 325 if username == "" { 326 username, err = readLine("Username: ", false) 327 if err != nil { 328 return "", "", err 329 } 330 username = strings.TrimSpace(username) 331 } 332 if username == "" { 333 password, err = readLine("Token: ", true) 334 if err != nil { 335 return "", "", err 336 } else if password == "" { 337 return "", "", goerrors.New("token required") 338 } 339 } else { 340 password, err = readLine("Password: ", true) 341 if err != nil { 342 return "", "", err 343 } else if password == "" { 344 return "", "", goerrors.New("password required") 345 } 346 } 347 } 348 349 return username, password, nil 350 } 351 352 // Copied/adapted from https://github.com/helm/helm 353 func readLine(prompt string, silent bool) (string, error) { 354 fmt.Print(prompt) 355 if silent { 356 fd := os.Stdin.Fd() 357 state, err := term.SaveState(fd) 358 if err != nil { 359 return "", err 360 } 361 err = term.DisableEcho(fd, state) 362 if err != nil { 363 return "", err 364 } 365 366 defer func() { 367 restoreErr := term.RestoreTerminal(fd, state) 368 if err == nil { 369 err = restoreErr 370 } 371 }() 372 } 373 374 reader := bufio.NewReader(os.Stdin) 375 line, _, err := reader.ReadLine() 376 if err != nil { 377 return "", err 378 } 379 if silent { 380 fmt.Println() 381 } 382 383 return string(line), nil 384 } 385 386 // FindKFiles will find all the '.k' files in the 'path' directory. 387 func FindKFiles(path string) ([]string, error) { 388 var files []string 389 info, err := os.Stat(path) 390 if err != nil { 391 return nil, err 392 } 393 if !info.IsDir() { 394 if strings.HasSuffix(path, ".k") { 395 files = append(files, path) 396 } 397 return files, nil 398 } 399 400 entries, err := os.ReadDir(path) 401 if err != nil { 402 return nil, err 403 } 404 for _, entry := range entries { 405 if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".k") { 406 files = append(files, filepath.Join(path, entry.Name())) 407 } 408 } 409 return files, nil 410 } 411 412 // RmNewline will remove all the '\r\n' and '\n' in the string 's'. 413 func RmNewline(s string) string { 414 return strings.ReplaceAll(strings.ReplaceAll(s, "\r\n", ""), "\n", "") 415 } 416 417 // JoinPath will join the 'elem' to the 'base' with '/'. 418 func JoinPath(base, elem string) string { 419 base = strings.TrimSuffix(base, "/") 420 elem = strings.TrimPrefix(elem, "/") 421 return base + "/" + elem 422 } 423 424 // IsUrl will check whether the string 'str' is a url. 425 func IsURL(str string) bool { 426 u, err := url.Parse(str) 427 return err == nil && u.Scheme != "" && u.Host != "" 428 } 429 430 // IsGitRepoUrl will check whether the string 'str' is a git repo url 431 func IsGitRepoUrl(str string) bool { 432 r := regexp.MustCompile(`((git|ssh|http(s)?)|(git@[\w\.]+))(:(//)?)([\w\.@\:/\-~]+)(\.git)?(/)?`) 433 return r.MatchString(str) 434 } 435 436 // IsRef will check whether the string 'str' is a reference. 437 func IsRef(str string) bool { 438 _, err := reference.ParseNormalizedNamed(str) 439 return err == nil 440 } 441 442 // IsTar will check whether the string 'str' is a tar path. 443 func IsTar(str string) bool { 444 return strings.HasSuffix(str, constants.TarPathSuffix) 445 } 446 447 // IsKfile will check whether the string 'str' is a k file path. 448 func IsKfile(str string) bool { 449 return strings.HasSuffix(str, constants.KFilePathSuffix) 450 } 451 452 // CheckPackageSum will check whether the 'checkedSum' is equal 453 // to the hash of the package under 'localPath'. 454 func CheckPackageSum(checkedSum, localPath string) bool { 455 if checkedSum == "" { 456 return false 457 } 458 459 sum, err := HashDir(localPath) 460 461 if err != nil { 462 return false 463 } 464 465 return checkedSum == sum 466 } 467 468 // AbsTarPath checks whether path 'tarPath' exists and whether path 'tarPath' ends with '.tar' 469 // And after checking, absTarPath return the abs path for 'tarPath'. 470 func AbsTarPath(tarPath string) (string, error) { 471 absTarPath, err := filepath.Abs(tarPath) 472 if err != nil { 473 return "", err 474 } 475 476 if filepath.Ext(absTarPath) != ".tar" { 477 return "", errors.InvalidKclPacakgeTar 478 } else if !DirExists(absTarPath) { 479 return "", errors.KclPacakgeTarNotFound 480 } 481 482 return absTarPath, nil 483 }