github.com/graemephi/kahugo@v0.62.3-0.20211121071557-d78c0423784d/hugofs/walk.go (about) 1 // Copyright 2019 The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package hugofs 15 16 import ( 17 "fmt" 18 "os" 19 "path/filepath" 20 "sort" 21 "strings" 22 23 "github.com/gohugoio/hugo/common/loggers" 24 25 "github.com/pkg/errors" 26 27 "github.com/spf13/afero" 28 ) 29 30 type ( 31 WalkFunc func(path string, info FileMetaInfo, err error) error 32 WalkHook func(dir FileMetaInfo, path string, readdir []FileMetaInfo) ([]FileMetaInfo, error) 33 ) 34 35 type Walkway struct { 36 fs afero.Fs 37 root string 38 basePath string 39 40 logger loggers.Logger 41 42 // May be pre-set 43 fi FileMetaInfo 44 dirEntries []FileMetaInfo 45 46 walkFn WalkFunc 47 walked bool 48 49 // We may traverse symbolic links and bite ourself. 50 seen map[string]bool 51 52 // Optional hooks 53 hookPre WalkHook 54 hookPost WalkHook 55 } 56 57 type WalkwayConfig struct { 58 Fs afero.Fs 59 Root string 60 BasePath string 61 62 Logger loggers.Logger 63 64 // One or both of these may be pre-set. 65 Info FileMetaInfo 66 DirEntries []FileMetaInfo 67 68 WalkFn WalkFunc 69 HookPre WalkHook 70 HookPost WalkHook 71 } 72 73 func NewWalkway(cfg WalkwayConfig) *Walkway { 74 var fs afero.Fs 75 if cfg.Info != nil { 76 fs = cfg.Info.Meta().Fs 77 } else { 78 fs = cfg.Fs 79 } 80 81 basePath := cfg.BasePath 82 if basePath != "" && !strings.HasSuffix(basePath, filepathSeparator) { 83 basePath += filepathSeparator 84 } 85 86 logger := cfg.Logger 87 if logger == nil { 88 logger = loggers.NewWarningLogger() 89 } 90 91 return &Walkway{ 92 fs: fs, 93 root: cfg.Root, 94 basePath: basePath, 95 fi: cfg.Info, 96 dirEntries: cfg.DirEntries, 97 walkFn: cfg.WalkFn, 98 hookPre: cfg.HookPre, 99 hookPost: cfg.HookPost, 100 logger: logger, 101 seen: make(map[string]bool), 102 } 103 } 104 105 func (w *Walkway) Walk() error { 106 if w.walked { 107 panic("this walkway is already walked") 108 } 109 w.walked = true 110 111 if w.fs == NoOpFs { 112 return nil 113 } 114 115 var fi FileMetaInfo 116 if w.fi != nil { 117 fi = w.fi 118 } else { 119 info, _, err := lstatIfPossible(w.fs, w.root) 120 if err != nil { 121 if os.IsNotExist(err) { 122 return nil 123 } 124 125 if w.checkErr(w.root, err) { 126 return nil 127 } 128 return w.walkFn(w.root, nil, errors.Wrapf(err, "walk: %q", w.root)) 129 } 130 fi = info.(FileMetaInfo) 131 } 132 133 if !fi.IsDir() { 134 return w.walkFn(w.root, nil, errors.New("file to walk must be a directory")) 135 } 136 137 return w.walk(w.root, fi, w.dirEntries, w.walkFn) 138 } 139 140 // if the filesystem supports it, use Lstat, else use fs.Stat 141 func lstatIfPossible(fs afero.Fs, path string) (os.FileInfo, bool, error) { 142 if lfs, ok := fs.(afero.Lstater); ok { 143 fi, b, err := lfs.LstatIfPossible(path) 144 return fi, b, err 145 } 146 fi, err := fs.Stat(path) 147 return fi, false, err 148 } 149 150 // checkErr returns true if the error is handled. 151 func (w *Walkway) checkErr(filename string, err error) bool { 152 if err == ErrPermissionSymlink { 153 logUnsupportedSymlink(filename, w.logger) 154 return true 155 } 156 157 if os.IsNotExist(err) { 158 // The file may be removed in process. 159 // This may be a ERROR situation, but it is not possible 160 // to determine as a general case. 161 w.logger.Warnf("File %q not found, skipping.", filename) 162 return true 163 } 164 165 return false 166 } 167 168 func logUnsupportedSymlink(filename string, logger loggers.Logger) { 169 logger.Warnf("Unsupported symlink found in %q, skipping.", filename) 170 } 171 172 // walk recursively descends path, calling walkFn. 173 // It follow symlinks if supported by the filesystem, but only the same path once. 174 func (w *Walkway) walk(path string, info FileMetaInfo, dirEntries []FileMetaInfo, walkFn WalkFunc) error { 175 err := walkFn(path, info, nil) 176 if err != nil { 177 if info.IsDir() && err == filepath.SkipDir { 178 return nil 179 } 180 return err 181 } 182 if !info.IsDir() { 183 return nil 184 } 185 186 meta := info.Meta() 187 filename := meta.Filename 188 189 if dirEntries == nil { 190 f, err := w.fs.Open(path) 191 if err != nil { 192 if w.checkErr(path, err) { 193 return nil 194 } 195 return walkFn(path, info, errors.Wrapf(err, "walk: open %q (%q)", path, w.root)) 196 } 197 198 fis, err := f.Readdir(-1) 199 f.Close() 200 if err != nil { 201 if w.checkErr(filename, err) { 202 return nil 203 } 204 return walkFn(path, info, errors.Wrap(err, "walk: Readdir")) 205 } 206 207 dirEntries = fileInfosToFileMetaInfos(fis) 208 209 if !meta.IsOrdered { 210 sort.Slice(dirEntries, func(i, j int) bool { 211 fii := dirEntries[i] 212 fij := dirEntries[j] 213 214 fim, fjm := fii.Meta(), fij.Meta() 215 216 // Pull bundle headers to the top. 217 ficlass, fjclass := fim.Classifier, fjm.Classifier 218 if ficlass != fjclass { 219 return ficlass < fjclass 220 } 221 222 // With multiple content dirs with different languages, 223 // there can be duplicate files, and a weight will be added 224 // to the closest one. 225 fiw, fjw := fim.Weight, fjm.Weight 226 if fiw != fjw { 227 return fiw > fjw 228 } 229 230 // Explicit order set. 231 fio, fjo := fim.Ordinal, fjm.Ordinal 232 if fio != fjo { 233 return fio < fjo 234 } 235 236 // When we walk into a symlink, we keep the reference to 237 // the original name. 238 fin, fjn := fim.Name, fjm.Name 239 if fin != "" && fjn != "" { 240 return fin < fjn 241 } 242 243 return fii.Name() < fij.Name() 244 }) 245 } 246 } 247 248 // First add some metadata to the dir entries 249 for _, fi := range dirEntries { 250 fim := fi.(FileMetaInfo) 251 252 meta := fim.Meta() 253 254 // Note that we use the original Name even if it's a symlink. 255 name := meta.Name 256 if name == "" { 257 name = fim.Name() 258 } 259 260 if name == "" { 261 panic(fmt.Sprintf("[%s] no name set in %v", path, meta)) 262 } 263 pathn := filepath.Join(path, name) 264 265 pathMeta := pathn 266 if w.basePath != "" { 267 pathMeta = strings.TrimPrefix(pathn, w.basePath) 268 } 269 270 meta.Path = normalizeFilename(pathMeta) 271 meta.PathWalk = pathn 272 273 if fim.IsDir() && w.isSeen(meta.Filename) { 274 // Prevent infinite recursion 275 // Possible cyclic reference 276 meta.SkipDir = true 277 } 278 } 279 280 if w.hookPre != nil { 281 dirEntries, err = w.hookPre(info, path, dirEntries) 282 if err != nil { 283 if err == filepath.SkipDir { 284 return nil 285 } 286 return err 287 } 288 } 289 290 for _, fi := range dirEntries { 291 fim := fi.(FileMetaInfo) 292 meta := fim.Meta() 293 294 if meta.SkipDir { 295 continue 296 } 297 298 err := w.walk(meta.PathWalk, fim, nil, walkFn) 299 if err != nil { 300 if !fi.IsDir() || err != filepath.SkipDir { 301 return err 302 } 303 } 304 } 305 306 if w.hookPost != nil { 307 dirEntries, err = w.hookPost(info, path, dirEntries) 308 if err != nil { 309 if err == filepath.SkipDir { 310 return nil 311 } 312 return err 313 } 314 } 315 return nil 316 } 317 318 func (w *Walkway) isSeen(filename string) bool { 319 if filename == "" { 320 return false 321 } 322 323 if w.seen[filename] { 324 return true 325 } 326 327 w.seen[filename] = true 328 return false 329 }