github.com/kovansky/hugo@v0.92.3-0.20220224232819-63076e4ff19f/helpers/path.go (about) 1 // Copyright 2019 The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package helpers 15 16 import ( 17 "errors" 18 "fmt" 19 "io" 20 "os" 21 "path/filepath" 22 "regexp" 23 "sort" 24 "strings" 25 "unicode" 26 27 "github.com/gohugoio/hugo/common/text" 28 29 "github.com/gohugoio/hugo/config" 30 31 "github.com/gohugoio/hugo/hugofs" 32 33 "github.com/gohugoio/hugo/common/hugio" 34 _errors "github.com/pkg/errors" 35 "github.com/spf13/afero" 36 ) 37 38 // MakePath takes a string with any characters and replace it 39 // so the string could be used in a path. 40 // It does so by creating a Unicode-sanitized string, with the spaces replaced, 41 // whilst preserving the original casing of the string. 42 // E.g. Social Media -> Social-Media 43 func (p *PathSpec) MakePath(s string) string { 44 return p.UnicodeSanitize(s) 45 } 46 47 // MakePathsSanitized applies MakePathSanitized on every item in the slice 48 func (p *PathSpec) MakePathsSanitized(paths []string) { 49 for i, path := range paths { 50 paths[i] = p.MakePathSanitized(path) 51 } 52 } 53 54 // MakePathSanitized creates a Unicode-sanitized string, with the spaces replaced 55 func (p *PathSpec) MakePathSanitized(s string) string { 56 if p.DisablePathToLower { 57 return p.MakePath(s) 58 } 59 return strings.ToLower(p.MakePath(s)) 60 } 61 62 // ToSlashTrimLeading is just a filepath.ToSlaas with an added / prefix trimmer. 63 func ToSlashTrimLeading(s string) string { 64 return strings.TrimPrefix(filepath.ToSlash(s), "/") 65 } 66 67 // MakeTitle converts the path given to a suitable title, trimming whitespace 68 // and replacing hyphens with whitespace. 69 func MakeTitle(inpath string) string { 70 return strings.Replace(strings.TrimSpace(inpath), "-", " ", -1) 71 } 72 73 // From https://golang.org/src/net/url/url.go 74 func ishex(c rune) bool { 75 switch { 76 case '0' <= c && c <= '9': 77 return true 78 case 'a' <= c && c <= 'f': 79 return true 80 case 'A' <= c && c <= 'F': 81 return true 82 } 83 return false 84 } 85 86 // UnicodeSanitize sanitizes string to be used in Hugo URL's, allowing only 87 // a predefined set of special Unicode characters. 88 // If RemovePathAccents configuration flag is enabled, Unicode accents 89 // are also removed. 90 // Hyphens in the original input are maintained. 91 // Spaces will be replaced with a single hyphen, and sequential replacement hyphens will be reduced to one. 92 func (p *PathSpec) UnicodeSanitize(s string) string { 93 if p.RemovePathAccents { 94 s = text.RemoveAccentsString(s) 95 } 96 97 source := []rune(s) 98 target := make([]rune, 0, len(source)) 99 var ( 100 prependHyphen bool 101 wasHyphen bool 102 ) 103 104 for i, r := range source { 105 isAllowed := r == '.' || r == '/' || r == '\\' || r == '_' || r == '#' || r == '+' || r == '~' || r == '-' 106 isAllowed = isAllowed || unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsMark(r) 107 isAllowed = isAllowed || (r == '%' && i+2 < len(source) && ishex(source[i+1]) && ishex(source[i+2])) 108 109 if isAllowed { 110 // track explicit hyphen in input; no need to add a new hyphen if 111 // we just saw one. 112 wasHyphen = r == '-' 113 114 if prependHyphen { 115 // if currently have a hyphen, don't prepend an extra one 116 if !wasHyphen { 117 target = append(target, '-') 118 } 119 prependHyphen = false 120 } 121 target = append(target, r) 122 } else if len(target) > 0 && !wasHyphen && unicode.IsSpace(r) { 123 prependHyphen = true 124 } 125 } 126 127 return string(target) 128 } 129 130 func makePathRelative(inPath string, possibleDirectories ...string) (string, error) { 131 for _, currentPath := range possibleDirectories { 132 if strings.HasPrefix(inPath, currentPath) { 133 return strings.TrimPrefix(inPath, currentPath), nil 134 } 135 } 136 return inPath, errors.New("can't extract relative path, unknown prefix") 137 } 138 139 // Should be good enough for Hugo. 140 var isFileRe = regexp.MustCompile(`.*\..{1,6}$`) 141 142 // GetDottedRelativePath expects a relative path starting after the content directory. 143 // It returns a relative path with dots ("..") navigating up the path structure. 144 func GetDottedRelativePath(inPath string) string { 145 inPath = filepath.Clean(filepath.FromSlash(inPath)) 146 147 if inPath == "." { 148 return "./" 149 } 150 151 if !isFileRe.MatchString(inPath) && !strings.HasSuffix(inPath, FilePathSeparator) { 152 inPath += FilePathSeparator 153 } 154 155 if !strings.HasPrefix(inPath, FilePathSeparator) { 156 inPath = FilePathSeparator + inPath 157 } 158 159 dir, _ := filepath.Split(inPath) 160 161 sectionCount := strings.Count(dir, FilePathSeparator) 162 163 if sectionCount == 0 || dir == FilePathSeparator { 164 return "./" 165 } 166 167 var dottedPath string 168 169 for i := 1; i < sectionCount; i++ { 170 dottedPath += "../" 171 } 172 173 return dottedPath 174 } 175 176 type NamedSlice struct { 177 Name string 178 Slice []string 179 } 180 181 func (n NamedSlice) String() string { 182 if len(n.Slice) == 0 { 183 return n.Name 184 } 185 return fmt.Sprintf("%s%s{%s}", n.Name, FilePathSeparator, strings.Join(n.Slice, ",")) 186 } 187 188 func ExtractAndGroupRootPaths(paths []string) []NamedSlice { 189 if len(paths) == 0 { 190 return nil 191 } 192 193 pathsCopy := make([]string, len(paths)) 194 hadSlashPrefix := strings.HasPrefix(paths[0], FilePathSeparator) 195 196 for i, p := range paths { 197 pathsCopy[i] = strings.Trim(filepath.ToSlash(p), "/") 198 } 199 200 sort.Strings(pathsCopy) 201 202 pathsParts := make([][]string, len(pathsCopy)) 203 204 for i, p := range pathsCopy { 205 pathsParts[i] = strings.Split(p, "/") 206 } 207 208 var groups [][]string 209 210 for i, p1 := range pathsParts { 211 c1 := -1 212 213 for j, p2 := range pathsParts { 214 if i == j { 215 continue 216 } 217 218 c2 := -1 219 220 for i, v := range p1 { 221 if i >= len(p2) { 222 break 223 } 224 if v != p2[i] { 225 break 226 } 227 228 c2 = i 229 } 230 231 if c1 == -1 || (c2 != -1 && c2 < c1) { 232 c1 = c2 233 } 234 } 235 236 if c1 != -1 { 237 groups = append(groups, p1[:c1+1]) 238 } else { 239 groups = append(groups, p1) 240 } 241 } 242 243 groupsStr := make([]string, len(groups)) 244 for i, g := range groups { 245 groupsStr[i] = strings.Join(g, "/") 246 } 247 248 groupsStr = UniqueStringsSorted(groupsStr) 249 250 var result []NamedSlice 251 252 for _, g := range groupsStr { 253 name := filepath.FromSlash(g) 254 if hadSlashPrefix { 255 name = FilePathSeparator + name 256 } 257 ns := NamedSlice{Name: name} 258 for _, p := range pathsCopy { 259 if !strings.HasPrefix(p, g) { 260 continue 261 } 262 263 p = strings.TrimPrefix(p, g) 264 if p != "" { 265 ns.Slice = append(ns.Slice, p) 266 } 267 } 268 269 ns.Slice = UniqueStrings(ExtractRootPaths(ns.Slice)) 270 271 result = append(result, ns) 272 } 273 274 return result 275 } 276 277 // ExtractRootPaths extracts the root paths from the supplied list of paths. 278 // The resulting root path will not contain any file separators, but there 279 // may be duplicates. 280 // So "/content/section/" becomes "content" 281 func ExtractRootPaths(paths []string) []string { 282 r := make([]string, len(paths)) 283 for i, p := range paths { 284 root := filepath.ToSlash(p) 285 sections := strings.Split(root, "/") 286 for _, section := range sections { 287 if section != "" { 288 root = section 289 break 290 } 291 } 292 r[i] = root 293 } 294 return r 295 } 296 297 // FindCWD returns the current working directory from where the Hugo 298 // executable is run. 299 func FindCWD() (string, error) { 300 serverFile, err := filepath.Abs(os.Args[0]) 301 if err != nil { 302 return "", fmt.Errorf("can't get absolute path for executable: %v", err) 303 } 304 305 path := filepath.Dir(serverFile) 306 realFile, err := filepath.EvalSymlinks(serverFile) 307 if err != nil { 308 if _, err = os.Stat(serverFile + ".exe"); err == nil { 309 realFile = filepath.Clean(serverFile + ".exe") 310 } 311 } 312 313 if err == nil && realFile != serverFile { 314 path = filepath.Dir(realFile) 315 } 316 317 return path, nil 318 } 319 320 // SymbolicWalk is like filepath.Walk, but it follows symbolic links. 321 func SymbolicWalk(fs afero.Fs, root string, walker hugofs.WalkFunc) error { 322 if _, isOs := fs.(*afero.OsFs); isOs { 323 // Mainly to track symlinks. 324 fs = hugofs.NewBaseFileDecorator(fs) 325 } 326 327 w := hugofs.NewWalkway(hugofs.WalkwayConfig{ 328 Fs: fs, 329 Root: root, 330 WalkFn: walker, 331 }) 332 333 return w.Walk() 334 } 335 336 // LstatIfPossible can be used to call Lstat if possible, else Stat. 337 func LstatIfPossible(fs afero.Fs, path string) (os.FileInfo, error) { 338 if lstater, ok := fs.(afero.Lstater); ok { 339 fi, _, err := lstater.LstatIfPossible(path) 340 return fi, err 341 } 342 343 return fs.Stat(path) 344 } 345 346 // SafeWriteToDisk is the same as WriteToDisk 347 // but it also checks to see if file/directory already exists. 348 func SafeWriteToDisk(inpath string, r io.Reader, fs afero.Fs) (err error) { 349 return afero.SafeWriteReader(fs, inpath, r) 350 } 351 352 // WriteToDisk writes content to disk. 353 func WriteToDisk(inpath string, r io.Reader, fs afero.Fs) (err error) { 354 return afero.WriteReader(fs, inpath, r) 355 } 356 357 // OpenFilesForWriting opens all the given filenames for writing. 358 func OpenFilesForWriting(fs afero.Fs, filenames ...string) (io.WriteCloser, error) { 359 var writeClosers []io.WriteCloser 360 for _, filename := range filenames { 361 f, err := OpenFileForWriting(fs, filename) 362 if err != nil { 363 for _, wc := range writeClosers { 364 wc.Close() 365 } 366 return nil, err 367 } 368 writeClosers = append(writeClosers, f) 369 } 370 371 return hugio.NewMultiWriteCloser(writeClosers...), nil 372 } 373 374 // OpenFileForWriting opens or creates the given file. If the target directory 375 // does not exist, it gets created. 376 func OpenFileForWriting(fs afero.Fs, filename string) (afero.File, error) { 377 filename = filepath.Clean(filename) 378 // Create will truncate if file already exists. 379 // os.Create will create any new files with mode 0666 (before umask). 380 f, err := fs.Create(filename) 381 if err != nil { 382 if !os.IsNotExist(err) { 383 return nil, err 384 } 385 if err = fs.MkdirAll(filepath.Dir(filename), 0777); err != nil { // before umask 386 return nil, err 387 } 388 f, err = fs.Create(filename) 389 } 390 391 return f, err 392 } 393 394 // GetCacheDir returns a cache dir from the given filesystem and config. 395 // The dir will be created if it does not exist. 396 func GetCacheDir(fs afero.Fs, cfg config.Provider) (string, error) { 397 cacheDir := getCacheDir(cfg) 398 if cacheDir != "" { 399 exists, err := DirExists(cacheDir, fs) 400 if err != nil { 401 return "", err 402 } 403 if !exists { 404 err := fs.MkdirAll(cacheDir, 0777) // Before umask 405 if err != nil { 406 return "", _errors.Wrap(err, "failed to create cache dir") 407 } 408 } 409 return cacheDir, nil 410 } 411 412 // Fall back to a cache in /tmp. 413 return GetTempDir("hugo_cache", fs), nil 414 } 415 416 func getCacheDir(cfg config.Provider) string { 417 // Always use the cacheDir config if set. 418 cacheDir := cfg.GetString("cacheDir") 419 if len(cacheDir) > 1 { 420 return addTrailingFileSeparator(cacheDir) 421 } 422 423 // See Issue #8714. 424 // Turns out that Cloudflare also sets NETLIFY=true in its build environment, 425 // but all of these 3 should not give any false positives. 426 if os.Getenv("NETLIFY") == "true" && os.Getenv("PULL_REQUEST") != "" && os.Getenv("DEPLOY_PRIME_URL") != "" { 427 // Netlify's cache behaviour is not documented, the currently best example 428 // is this project: 429 // https://github.com/philhawksworth/content-shards/blob/master/gulpfile.js 430 return "/opt/build/cache/hugo_cache/" 431 } 432 433 // This will fall back to an hugo_cache folder in the tmp dir, which should work fine for most CI 434 // providers. See this for a working CircleCI setup: 435 // https://github.com/bep/hugo-sass-test/blob/6c3960a8f4b90e8938228688bc49bdcdd6b2d99e/.circleci/config.yml 436 // If not, they can set the HUGO_CACHEDIR environment variable or cacheDir config key. 437 return "" 438 } 439 440 func addTrailingFileSeparator(s string) string { 441 if !strings.HasSuffix(s, FilePathSeparator) { 442 s = s + FilePathSeparator 443 } 444 return s 445 } 446 447 // GetTempDir returns a temporary directory with the given sub path. 448 func GetTempDir(subPath string, fs afero.Fs) string { 449 return afero.GetTempDir(fs, subPath) 450 } 451 452 // DirExists checks if a path exists and is a directory. 453 func DirExists(path string, fs afero.Fs) (bool, error) { 454 return afero.DirExists(fs, path) 455 } 456 457 // IsDir checks if a given path is a directory. 458 func IsDir(path string, fs afero.Fs) (bool, error) { 459 return afero.IsDir(fs, path) 460 } 461 462 // IsEmpty checks if a given path is empty. 463 func IsEmpty(path string, fs afero.Fs) (bool, error) { 464 return afero.IsEmpty(fs, path) 465 } 466 467 // Exists checks if a file or directory exists. 468 func Exists(path string, fs afero.Fs) (bool, error) { 469 return afero.Exists(fs, path) 470 } 471 472 // AddTrailingSlash adds a trailing Unix styled slash (/) if not already 473 // there. 474 func AddTrailingSlash(path string) string { 475 if !strings.HasSuffix(path, "/") { 476 path += "/" 477 } 478 return path 479 }