github.com/jaylevin/jenkins-library@v1.230.4/pkg/piperutils/FileUtils.go (about) 1 package piperutils 2 3 import ( 4 "archive/tar" 5 "archive/zip" 6 "compress/gzip" 7 "crypto/sha256" 8 "errors" 9 "fmt" 10 "io" 11 "io/fs" 12 "io/ioutil" 13 "os" 14 "path/filepath" 15 "strings" 16 "time" 17 18 "github.com/bmatcuk/doublestar" 19 ) 20 21 // FileUtils ... 22 type FileUtils interface { 23 Abs(path string) (string, error) 24 DirExists(path string) (bool, error) 25 FileExists(filename string) (bool, error) 26 Copy(src, dest string) (int64, error) 27 Move(src, dest string) error 28 FileRead(path string) ([]byte, error) 29 FileWrite(path string, content []byte, perm os.FileMode) error 30 FileRemove(path string) error 31 MkdirAll(path string, perm os.FileMode) error 32 Chmod(path string, mode os.FileMode) error 33 Glob(pattern string) (matches []string, err error) 34 Chdir(path string) error 35 TempDir(string, string) (string, error) 36 RemoveAll(string) error 37 FileRename(string, string) error 38 Getwd() (string, error) 39 Symlink(oldname string, newname string) error 40 SHA256(path string) (string, error) 41 CurrentTime(format string) string 42 Open(name string) (io.ReadWriteCloser, error) 43 Create(name string) (io.ReadWriteCloser, error) 44 } 45 46 // Files ... 47 type Files struct { 48 } 49 50 // TempDir creates a temporary directory 51 func (f Files) TempDir(dir, pattern string) (name string, err error) { 52 if len(dir) == 0 { 53 // lazy init system temp dir in case it doesn't exist 54 if exists, _ := f.DirExists(os.TempDir()); !exists { 55 f.MkdirAll(os.TempDir(), 0666) 56 } 57 } 58 59 return ioutil.TempDir(dir, pattern) 60 } 61 62 // FileExists returns true if the file system entry for the given path exists and is not a directory. 63 func (f Files) FileExists(filename string) (bool, error) { 64 info, err := os.Stat(filename) 65 66 if os.IsNotExist(err) { 67 return false, nil 68 } 69 if err != nil { 70 return false, err 71 } 72 73 return !info.IsDir(), nil 74 } 75 76 // FileExists returns true if the file system entry for the given path exists and is not a directory. 77 func FileExists(filename string) (bool, error) { 78 return Files{}.FileExists(filename) 79 } 80 81 // DirExists returns true if the file system entry for the given path exists and is a directory. 82 func (f Files) DirExists(path string) (bool, error) { 83 info, err := os.Stat(path) 84 85 if os.IsNotExist(err) { 86 return false, nil 87 } 88 if err != nil { 89 return false, err 90 } 91 92 return info.IsDir(), nil 93 } 94 95 // Copy ... 96 func (f Files) Copy(src, dst string) (int64, error) { 97 98 exists, err := f.FileExists(src) 99 100 if err != nil { 101 return 0, err 102 } 103 104 if !exists { 105 return 0, errors.New("Source file '" + src + "' does not exist") 106 } 107 108 source, err := os.Open(src) 109 if err != nil { 110 return 0, err 111 } 112 defer func() { _ = source.Close() }() 113 114 destination, err := os.Create(dst) 115 if err != nil { 116 return 0, err 117 } 118 stats, err := os.Stat(src) 119 if err != nil { 120 return 0, err 121 } 122 123 os.Chmod(dst, stats.Mode()) 124 defer func() { _ = destination.Close() }() 125 nBytes, err := CopyData(destination, source) 126 return nBytes, err 127 } 128 129 func (f Files) Move(src, dst string) error { 130 if exists, err := f.FileExists(src); err != nil { 131 return err 132 } else if !exists { 133 return fmt.Errorf("file doesn't exist: %s", src) 134 } 135 136 if _, err := f.Copy(src, dst); err != nil { 137 return err 138 } 139 140 return f.FileRemove(src) 141 } 142 143 //Chmod is a wrapper for os.Chmod(). 144 func (f Files) Chmod(path string, mode os.FileMode) error { 145 return os.Chmod(path, mode) 146 } 147 148 // Unzip will decompress a zip archive, moving all files and folders 149 // within the zip file (parameter 1) to an output directory (parameter 2). 150 // from https://golangcode.com/unzip-files-in-go/ with the following license: 151 // MIT License 152 // 153 // Copyright (c) 2017 Edd Turtle 154 // 155 // Permission is hereby granted, free of charge, to any person obtaining a copy 156 // of this software and associated documentation files (the "Software"), to deal 157 // in the Software without restriction, including without limitation the rights 158 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 159 // copies of the Software, and to permit persons to whom the Software is 160 // furnished to do so, subject to the following conditions: 161 // 162 // The above copyright notice and this permission notice shall be included in all 163 // copies or substantial portions of the Software. 164 // 165 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 166 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 167 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 168 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 169 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 170 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 171 // SOFTWARE. 172 func Unzip(src, dest string) ([]string, error) { 173 174 var filenames []string 175 176 r, err := zip.OpenReader(src) 177 if err != nil { 178 return filenames, err 179 } 180 defer func() { _ = r.Close() }() 181 182 for _, f := range r.File { 183 184 // Store filename/path for returning and using later on 185 fpath := filepath.Join(dest, f.Name) 186 187 // Check for ZipSlip. More Info: http://bit.ly/2MsjAWE 188 if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) { 189 return filenames, fmt.Errorf("%s: illegal file path", fpath) 190 } 191 192 filenames = append(filenames, fpath) 193 194 if f.FileInfo().IsDir() { 195 // Make Folder 196 err := os.MkdirAll(fpath, os.ModePerm) 197 if err != nil { 198 return filenames, fmt.Errorf("failed to create directory: %w", err) 199 } 200 continue 201 } 202 203 // Make File 204 if err = os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil { 205 return filenames, err 206 } 207 208 outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) 209 if err != nil { 210 return filenames, err 211 } 212 213 rc, err := f.Open() 214 if err != nil { 215 return filenames, err 216 } 217 218 _, err = CopyData(outFile, rc) 219 220 // Close the file without defer to close before next iteration of loop 221 _ = outFile.Close() 222 _ = rc.Close() 223 224 if err != nil { 225 return filenames, err 226 } 227 } 228 return filenames, nil 229 } 230 231 // Untar will decompress a gzipped archive and then untar it, moving all files and folders 232 // within the tgz file (parameter 1) to an output directory (parameter 2). 233 // some tar like the one created from npm have an addtional package folder which need to be removed during untar 234 // stripComponent level acts the same like in the tar cli with level 1 corresponding to elimination of parent folder 235 // stripComponentLevel = 1 -> parentFolder/someFile.Txt -> someFile.Txt 236 // stripComponentLevel = 2 -> parentFolder/childFolder/someFile.Txt -> someFile.Txt 237 // when stripCompenent in 0 the untar will retain the original tar folder structure 238 // when stripCompmenet is greater than 0 the expectation is all files must be under that level folder and if not there is a hard check and failure condition 239 240 func Untar(src string, dest string, stripComponentLevel int) error { 241 file, err := os.Open(src) 242 defer file.Close() 243 244 if err != nil { 245 return fmt.Errorf("unable to open src: %v", err) 246 } 247 248 if b, err := isFileGzipped(src); err == nil && b { 249 zr, err := gzip.NewReader(file) 250 251 if err != nil { 252 return fmt.Errorf("requires gzip-compressed body: %v", err) 253 } 254 255 return untar(zr, dest, stripComponentLevel) 256 } 257 258 return untar(file, dest, stripComponentLevel) 259 } 260 261 func untar(r io.Reader, dir string, level int) (err error) { 262 madeDir := map[string]bool{} 263 264 tr := tar.NewReader(r) 265 for { 266 f, err := tr.Next() 267 if err == io.EOF { 268 break 269 } 270 if err != nil { 271 return fmt.Errorf("tar error: %v", err) 272 } 273 if strings.HasPrefix(f.Name, "/") { 274 f.Name = fmt.Sprintf(".%s", f.Name) 275 } 276 if !validRelPath(f.Name) { // blocks path traversal attacks 277 return fmt.Errorf("tar contained invalid name error %q", f.Name) 278 } 279 rel := filepath.FromSlash(f.Name) 280 281 // when level X folder(s) needs to be removed we first check that the rel path must have atleast X or greater than X pathseperatorserr 282 // or else we might end in index out of range 283 if level > 0 { 284 if strings.Count(rel, string(os.PathSeparator)) >= level { 285 relSplit := strings.SplitN(rel, string(os.PathSeparator), level+1) 286 rel = relSplit[level] 287 } else { 288 return fmt.Errorf("files %q in tarball archive not under level %v", f.Name, level) 289 } 290 } 291 292 abs := filepath.Join(dir, rel) 293 294 fi := f.FileInfo() 295 mode := fi.Mode() 296 switch { 297 case mode.IsRegular(): 298 // Make the directory. This is redundant because it should 299 // already be made by a directory entry in the tar 300 // beforehand. Thus, don't check for errors; the next 301 // write will fail with the same error. 302 dir := filepath.Dir(abs) 303 if !madeDir[dir] { 304 if err := os.MkdirAll(filepath.Dir(abs), 0755); err != nil { 305 return err 306 } 307 madeDir[dir] = true 308 } 309 wf, err := os.OpenFile(abs, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode.Perm()) 310 if err != nil { 311 return err 312 } 313 n, err := io.Copy(wf, tr) 314 if closeErr := wf.Close(); closeErr != nil && err == nil { 315 err = closeErr 316 } 317 if err != nil { 318 return fmt.Errorf("error writing to %s: %v", abs, err) 319 } 320 if n != f.Size { 321 return fmt.Errorf("only wrote %d bytes to %s; expected %d", n, abs, f.Size) 322 } 323 case mode.IsDir(): 324 if err := os.MkdirAll(abs, 0755); err != nil { 325 return err 326 } 327 madeDir[abs] = true 328 case mode&fs.ModeSymlink != 0: 329 if err := os.Symlink(f.Linkname, abs); err != nil { 330 return err 331 } 332 default: 333 return fmt.Errorf("tar file entry %s contained unsupported file type %v", f.Name, mode) 334 } 335 } 336 return nil 337 } 338 339 // isFileGzipped checks the first 3 bytes of the given file to determine if it is gzipped or not. Returns `true` if the file is gzipped. 340 func isFileGzipped(file string) (bool, error) { 341 f, err := os.Open(file) 342 defer f.Close() 343 344 if err != nil { 345 return false, err 346 } 347 348 b := make([]byte, 3) 349 _, err = io.ReadFull(f, b) 350 351 if err != nil { 352 return false, err 353 } 354 355 return b[0] == 0x1f && b[1] == 0x8b && b[2] == 8, nil 356 } 357 358 func validRelPath(p string) bool { 359 if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") { 360 return false 361 } 362 return true 363 } 364 365 // Copy ... 366 func Copy(src, dst string) (int64, error) { 367 return Files{}.Copy(src, dst) 368 } 369 370 // FileRead is a wrapper for ioutil.ReadFile(). 371 func (f Files) FileRead(path string) ([]byte, error) { 372 return ioutil.ReadFile(path) 373 } 374 375 // FileWrite is a wrapper for ioutil.WriteFile(). 376 func (f Files) FileWrite(path string, content []byte, perm os.FileMode) error { 377 return ioutil.WriteFile(path, content, perm) 378 } 379 380 // FileRemove is a wrapper for os.Remove(). 381 func (f Files) FileRemove(path string) error { 382 return os.Remove(path) 383 } 384 385 // FileRename is a wrapper for os.Rename(). 386 func (f Files) FileRename(oldPath, newPath string) error { 387 return os.Rename(oldPath, newPath) 388 } 389 390 // FileOpen is a wrapper for os.OpenFile(). 391 func (f *Files) FileOpen(name string, flag int, perm os.FileMode) (*os.File, error) { 392 return os.OpenFile(name, flag, perm) 393 } 394 395 // MkdirAll is a wrapper for os.MkdirAll(). 396 func (f Files) MkdirAll(path string, perm os.FileMode) error { 397 return os.MkdirAll(path, perm) 398 } 399 400 // RemoveAll is a wrapper for os.RemoveAll(). 401 func (f Files) RemoveAll(path string) error { 402 return os.RemoveAll(path) 403 } 404 405 // Glob is a wrapper for doublestar.Glob(). 406 func (f Files) Glob(pattern string) (matches []string, err error) { 407 return doublestar.Glob(pattern) 408 } 409 410 // ExcludeFiles returns a slice of files, which contains only the sub-set of files that matched none 411 // of the glob patterns in the provided excludes list. 412 func ExcludeFiles(files, excludes []string) ([]string, error) { 413 if len(excludes) == 0 { 414 return files, nil 415 } 416 417 var filteredFiles []string 418 for _, file := range files { 419 includeFile := true 420 file = filepath.FromSlash(file) 421 for _, exclude := range excludes { 422 matched, err := doublestar.PathMatch(exclude, file) 423 if err != nil { 424 return nil, fmt.Errorf("failed to match file %s to pattern %s: %w", file, exclude, err) 425 } 426 if matched { 427 includeFile = false 428 break 429 } 430 } 431 if includeFile { 432 filteredFiles = append(filteredFiles, file) 433 } 434 } 435 436 return filteredFiles, nil 437 } 438 439 // Getwd is a wrapper for os.Getwd(). 440 func (f Files) Getwd() (string, error) { 441 return os.Getwd() 442 } 443 444 // Chdir is a wrapper for os.Chdir(). 445 func (f Files) Chdir(path string) error { 446 return os.Chdir(path) 447 } 448 449 // Stat is a wrapper for os.Stat() 450 func (f Files) Stat(path string) (os.FileInfo, error) { 451 return os.Stat(path) 452 } 453 454 // Abs is a wrapper for filepath.Abs() 455 func (f Files) Abs(path string) (string, error) { 456 return filepath.Abs(path) 457 } 458 459 // Symlink is a wrapper for os.Symlink 460 func (f Files) Symlink(oldname, newname string) error { 461 return os.Symlink(oldname, newname) 462 } 463 464 // SHA256 computes a SHA256 for a given file 465 func (f Files) SHA256(path string) (string, error) { 466 file, err := os.Open(path) 467 if err != nil { 468 return "", err 469 } 470 defer file.Close() 471 472 hash := sha256.New() 473 _, err = io.Copy(hash, file) 474 if err != nil { 475 return "", err 476 } 477 478 return fmt.Sprintf("%x", string(hash.Sum(nil))), nil 479 } 480 481 // CurrentTime returns the current time in the specified format 482 func (f Files) CurrentTime(format string) string { 483 fString := format 484 if len(format) == 0 { 485 fString = "20060102-150405" 486 } 487 return fmt.Sprint(time.Now().Format(fString)) 488 } 489 490 // Open is a wrapper for os.Open 491 func (f Files) Open(name string) (io.ReadWriteCloser, error) { 492 return os.Open(name) 493 } 494 495 // Create is a wrapper for os.Create 496 func (f Files) Create(name string) (io.ReadWriteCloser, error) { 497 return os.Create(name) 498 }