github.com/kovansky/hugo@v0.92.3-0.20220224232819-63076e4ff19f/hugofs/rootmapping_fs.go (about) 1 // Copyright 2019 The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package hugofs 15 16 import ( 17 "fmt" 18 "os" 19 "path/filepath" 20 "strings" 21 22 "github.com/gohugoio/hugo/hugofs/files" 23 24 "github.com/pkg/errors" 25 26 radix "github.com/armon/go-radix" 27 "github.com/spf13/afero" 28 ) 29 30 var filepathSeparator = string(filepath.Separator) 31 32 // NewRootMappingFs creates a new RootMappingFs on top of the provided with 33 // root mappings with some optional metadata about the root. 34 // Note that From represents a virtual root that maps to the actual filename in To. 35 func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { 36 rootMapToReal := radix.New() 37 var virtualRoots []RootMapping 38 39 for _, rm := range rms { 40 (&rm).clean() 41 42 fromBase := files.ResolveComponentFolder(rm.From) 43 44 if len(rm.To) < 2 { 45 panic(fmt.Sprintf("invalid root mapping; from/to: %s/%s", rm.From, rm.To)) 46 } 47 48 fi, err := fs.Stat(rm.To) 49 if err != nil { 50 if os.IsNotExist(err) { 51 continue 52 } 53 return nil, err 54 } 55 // Extract "blog" from "content/blog" 56 rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator) 57 if rm.Meta == nil { 58 rm.Meta = NewFileMeta() 59 } 60 61 rm.Meta.SourceRoot = rm.To 62 rm.Meta.BaseDir = rm.ToBasedir 63 rm.Meta.MountRoot = rm.path 64 rm.Meta.Module = rm.Module 65 rm.Meta.IsProject = rm.IsProject 66 67 meta := rm.Meta.Copy() 68 69 if !fi.IsDir() { 70 _, name := filepath.Split(rm.From) 71 meta.Name = name 72 } 73 74 rm.fi = NewFileMetaInfo(fi, meta) 75 76 key := filepathSeparator + rm.From 77 var mappings []RootMapping 78 v, found := rootMapToReal.Get(key) 79 if found { 80 // There may be more than one language pointing to the same root. 81 mappings = v.([]RootMapping) 82 } 83 mappings = append(mappings, rm) 84 rootMapToReal.Insert(key, mappings) 85 86 virtualRoots = append(virtualRoots, rm) 87 } 88 89 rootMapToReal.Insert(filepathSeparator, virtualRoots) 90 91 rfs := &RootMappingFs{ 92 Fs: fs, 93 rootMapToReal: rootMapToReal, 94 } 95 96 return rfs, nil 97 } 98 99 func newRootMappingFsFromFromTo( 100 baseDir string, 101 fs afero.Fs, 102 fromTo ...string, 103 ) (*RootMappingFs, error) { 104 rms := make([]RootMapping, len(fromTo)/2) 105 for i, j := 0, 0; j < len(fromTo); i, j = i+1, j+2 { 106 rms[i] = RootMapping{ 107 From: fromTo[j], 108 To: fromTo[j+1], 109 ToBasedir: baseDir, 110 } 111 } 112 113 return NewRootMappingFs(fs, rms...) 114 } 115 116 // RootMapping describes a virtual file or directory mount. 117 type RootMapping struct { 118 From string // The virtual mount. 119 To string // The source directory or file. 120 ToBasedir string // The base of To. May be empty if an absolute path was provided. 121 Module string // The module path/ID. 122 IsProject bool // Whether this is a mount in the main project. 123 Meta *FileMeta // File metadata (lang etc.) 124 125 fi FileMetaInfo 126 path string // The virtual mount point, e.g. "blog". 127 128 } 129 130 type keyRootMappings struct { 131 key string 132 roots []RootMapping 133 } 134 135 func (rm *RootMapping) clean() { 136 rm.From = strings.Trim(filepath.Clean(rm.From), filepathSeparator) 137 rm.To = filepath.Clean(rm.To) 138 } 139 140 func (r RootMapping) filename(name string) string { 141 if name == "" { 142 return r.To 143 } 144 return filepath.Join(r.To, strings.TrimPrefix(name, r.From)) 145 } 146 147 func (r RootMapping) trimFrom(name string) string { 148 if name == "" { 149 return "" 150 } 151 return strings.TrimPrefix(name, r.From) 152 } 153 154 // A RootMappingFs maps several roots into one. Note that the root of this filesystem 155 // is directories only, and they will be returned in Readdir and Readdirnames 156 // in the order given. 157 type RootMappingFs struct { 158 afero.Fs 159 rootMapToReal *radix.Tree 160 } 161 162 func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) { 163 base = filepathSeparator + fs.cleanName(base) 164 roots := fs.getRootsWithPrefix(base) 165 166 if roots == nil { 167 return nil, nil 168 } 169 170 fss := make([]FileMetaInfo, len(roots)) 171 for i, r := range roots { 172 bfs := afero.NewBasePathFs(fs.Fs, r.To) 173 bfs = decoratePath(bfs, func(name string) string { 174 p := strings.TrimPrefix(name, r.To) 175 if r.path != "" { 176 // Make sure it's mounted to a any sub path, e.g. blog 177 p = filepath.Join(r.path, p) 178 } 179 p = strings.TrimLeft(p, filepathSeparator) 180 return p 181 }) 182 183 fs := bfs 184 if r.Meta.InclusionFilter != nil { 185 fs = newFilenameFilterFs(fs, r.To, r.Meta.InclusionFilter) 186 } 187 fs = decorateDirs(fs, r.Meta) 188 fi, err := fs.Stat("") 189 if err != nil { 190 return nil, errors.Wrap(err, "RootMappingFs.Dirs") 191 } 192 193 if !fi.IsDir() { 194 fi.(FileMetaInfo).Meta().Merge(r.Meta) 195 } 196 197 fss[i] = fi.(FileMetaInfo) 198 } 199 200 return fss, nil 201 } 202 203 // Filter creates a copy of this filesystem with only mappings matching a filter. 204 func (fs RootMappingFs) Filter(f func(m RootMapping) bool) *RootMappingFs { 205 rootMapToReal := radix.New() 206 fs.rootMapToReal.Walk(func(b string, v interface{}) bool { 207 rms := v.([]RootMapping) 208 var nrms []RootMapping 209 for _, rm := range rms { 210 if f(rm) { 211 nrms = append(nrms, rm) 212 } 213 } 214 if len(nrms) != 0 { 215 rootMapToReal.Insert(b, nrms) 216 } 217 return false 218 }) 219 220 fs.rootMapToReal = rootMapToReal 221 222 return &fs 223 } 224 225 // LstatIfPossible returns the os.FileInfo structure describing a given file. 226 func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { 227 fis, err := fs.doLstat(name) 228 if err != nil { 229 return nil, false, err 230 } 231 return fis[0], false, nil 232 } 233 234 // Open opens the named file for reading. 235 func (fs *RootMappingFs) Open(name string) (afero.File, error) { 236 fis, err := fs.doLstat(name) 237 if err != nil { 238 return nil, err 239 } 240 241 return fs.newUnionFile(fis...) 242 } 243 244 // Stat returns the os.FileInfo structure describing a given file. If there is 245 // an error, it will be of type *os.PathError. 246 func (fs *RootMappingFs) Stat(name string) (os.FileInfo, error) { 247 fi, _, err := fs.LstatIfPossible(name) 248 return fi, err 249 } 250 251 func (fs *RootMappingFs) hasPrefix(prefix string) bool { 252 hasPrefix := false 253 fs.rootMapToReal.WalkPrefix(prefix, func(b string, v interface{}) bool { 254 hasPrefix = true 255 return true 256 }) 257 258 return hasPrefix 259 } 260 261 func (fs *RootMappingFs) getRoot(key string) []RootMapping { 262 v, found := fs.rootMapToReal.Get(key) 263 if !found { 264 return nil 265 } 266 267 return v.([]RootMapping) 268 } 269 270 func (fs *RootMappingFs) getRoots(key string) (string, []RootMapping) { 271 s, v, found := fs.rootMapToReal.LongestPrefix(key) 272 if !found || (s == filepathSeparator && key != filepathSeparator) { 273 return "", nil 274 } 275 return s, v.([]RootMapping) 276 } 277 278 func (fs *RootMappingFs) debug() { 279 fmt.Println("debug():") 280 fs.rootMapToReal.Walk(func(s string, v interface{}) bool { 281 fmt.Println("Key", s) 282 return false 283 }) 284 } 285 286 func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []RootMapping { 287 var roots []RootMapping 288 fs.rootMapToReal.WalkPrefix(prefix, func(b string, v interface{}) bool { 289 roots = append(roots, v.([]RootMapping)...) 290 return false 291 }) 292 293 return roots 294 } 295 296 func (fs *RootMappingFs) getAncestors(prefix string) []keyRootMappings { 297 var roots []keyRootMappings 298 fs.rootMapToReal.WalkPath(prefix, func(s string, v interface{}) bool { 299 if strings.HasPrefix(prefix, s+filepathSeparator) { 300 roots = append(roots, keyRootMappings{ 301 key: s, 302 roots: v.([]RootMapping), 303 }) 304 } 305 return false 306 }) 307 308 return roots 309 } 310 311 func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) { 312 meta := fis[0].Meta() 313 f, err := meta.Open() 314 if err != nil { 315 return nil, err 316 } 317 if len(fis) == 1 { 318 return f, nil 319 } 320 321 rf := &rootMappingFile{File: f, fs: fs, name: meta.Name, meta: meta} 322 if len(fis) == 1 { 323 return rf, err 324 } 325 326 next, err := fs.newUnionFile(fis[1:]...) 327 if err != nil { 328 return nil, err 329 } 330 331 uf := &afero.UnionFile{Base: rf, Layer: next} 332 333 uf.Merger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) { 334 // Ignore duplicate directory entries 335 seen := make(map[string]bool) 336 var result []os.FileInfo 337 338 for _, fis := range [][]os.FileInfo{bofi, lofi} { 339 for _, fi := range fis { 340 341 if fi.IsDir() && seen[fi.Name()] { 342 continue 343 } 344 345 if fi.IsDir() { 346 seen[fi.Name()] = true 347 } 348 349 result = append(result, fi) 350 } 351 } 352 353 return result, nil 354 } 355 356 return uf, nil 357 } 358 359 func (fs *RootMappingFs) cleanName(name string) string { 360 return strings.Trim(filepath.Clean(name), filepathSeparator) 361 } 362 363 func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error) { 364 prefix = filepathSeparator + fs.cleanName(prefix) 365 366 var fis []os.FileInfo 367 368 seen := make(map[string]bool) // Prevent duplicate directories 369 level := strings.Count(prefix, filepathSeparator) 370 371 collectDir := func(rm RootMapping, fi FileMetaInfo) error { 372 f, err := fi.Meta().Open() 373 if err != nil { 374 return err 375 } 376 direntries, err := f.Readdir(-1) 377 if err != nil { 378 f.Close() 379 return err 380 } 381 382 for _, fi := range direntries { 383 meta := fi.(FileMetaInfo).Meta() 384 meta.Merge(rm.Meta) 385 if !rm.Meta.InclusionFilter.Match(strings.TrimPrefix(meta.Filename, meta.SourceRoot), fi.IsDir()) { 386 continue 387 } 388 389 if fi.IsDir() { 390 name := fi.Name() 391 if seen[name] { 392 continue 393 } 394 seen[name] = true 395 opener := func() (afero.File, error) { 396 return fs.Open(filepath.Join(rm.From, name)) 397 } 398 fi = newDirNameOnlyFileInfo(name, meta, opener) 399 } 400 401 fis = append(fis, fi) 402 } 403 404 f.Close() 405 406 return nil 407 } 408 409 // First add any real files/directories. 410 rms := fs.getRoot(prefix) 411 for _, rm := range rms { 412 if err := collectDir(rm, rm.fi); err != nil { 413 return nil, err 414 } 415 } 416 417 // Next add any file mounts inside the given directory. 418 prefixInside := prefix + filepathSeparator 419 fs.rootMapToReal.WalkPrefix(prefixInside, func(s string, v interface{}) bool { 420 if (strings.Count(s, filepathSeparator) - level) != 1 { 421 // This directory is not part of the current, but we 422 // need to include the first name part to make it 423 // navigable. 424 path := strings.TrimPrefix(s, prefixInside) 425 parts := strings.Split(path, filepathSeparator) 426 name := parts[0] 427 428 if seen[name] { 429 return false 430 } 431 seen[name] = true 432 opener := func() (afero.File, error) { 433 return fs.Open(path) 434 } 435 436 fi := newDirNameOnlyFileInfo(name, nil, opener) 437 fis = append(fis, fi) 438 439 return false 440 } 441 442 rms := v.([]RootMapping) 443 for _, rm := range rms { 444 if !rm.fi.IsDir() { 445 // A single file mount 446 fis = append(fis, rm.fi) 447 continue 448 } 449 name := filepath.Base(rm.From) 450 if seen[name] { 451 continue 452 } 453 seen[name] = true 454 455 opener := func() (afero.File, error) { 456 return fs.Open(rm.From) 457 } 458 459 fi := newDirNameOnlyFileInfo(name, rm.Meta, opener) 460 461 fis = append(fis, fi) 462 463 } 464 465 return false 466 }) 467 468 // Finally add any ancestor dirs with files in this directory. 469 ancestors := fs.getAncestors(prefix) 470 for _, root := range ancestors { 471 subdir := strings.TrimPrefix(prefix, root.key) 472 for _, rm := range root.roots { 473 if rm.fi.IsDir() { 474 fi, err := rm.fi.Meta().JoinStat(subdir) 475 if err == nil { 476 if err := collectDir(rm, fi); err != nil { 477 return nil, err 478 } 479 } 480 } 481 } 482 } 483 484 return fis, nil 485 } 486 487 func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) { 488 name = fs.cleanName(name) 489 key := filepathSeparator + name 490 491 roots := fs.getRoot(key) 492 493 if roots == nil { 494 if fs.hasPrefix(key) { 495 // We have directories mounted below this. 496 // Make it look like a directory. 497 return []FileMetaInfo{newDirNameOnlyFileInfo(name, nil, fs.virtualDirOpener(name))}, nil 498 } 499 500 // Find any real files or directories with this key. 501 _, roots := fs.getRoots(key) 502 if roots == nil { 503 return nil, &os.PathError{Op: "LStat", Path: name, Err: os.ErrNotExist} 504 } 505 506 var err error 507 var fis []FileMetaInfo 508 509 for _, rm := range roots { 510 var fi FileMetaInfo 511 fi, _, err = fs.statRoot(rm, name) 512 if err == nil { 513 fis = append(fis, fi) 514 } 515 } 516 517 if fis != nil { 518 return fis, nil 519 } 520 521 if err == nil { 522 err = &os.PathError{Op: "LStat", Path: name, Err: err} 523 } 524 525 return nil, err 526 } 527 528 fileCount := 0 529 var wasFiltered bool 530 for _, root := range roots { 531 meta := root.fi.Meta() 532 if !meta.InclusionFilter.Match(strings.TrimPrefix(meta.Filename, meta.SourceRoot), root.fi.IsDir()) { 533 wasFiltered = true 534 continue 535 } 536 537 if !root.fi.IsDir() { 538 fileCount++ 539 } 540 if fileCount > 1 { 541 break 542 } 543 } 544 545 if fileCount == 0 { 546 if wasFiltered { 547 return nil, os.ErrNotExist 548 } 549 // Dir only. 550 return []FileMetaInfo{newDirNameOnlyFileInfo(name, roots[0].Meta, fs.virtualDirOpener(name))}, nil 551 } 552 553 if fileCount > 1 { 554 // Not supported by this filesystem. 555 return nil, errors.Errorf("found multiple files with name %q, use .Readdir or the source filesystem directly", name) 556 } 557 558 return []FileMetaInfo{roots[0].fi}, nil 559 } 560 561 func (fs *RootMappingFs) statRoot(root RootMapping, name string) (FileMetaInfo, bool, error) { 562 if !root.Meta.InclusionFilter.Match(root.trimFrom(name), root.fi.IsDir()) { 563 return nil, false, os.ErrNotExist 564 } 565 filename := root.filename(name) 566 567 fi, b, err := lstatIfPossible(fs.Fs, filename) 568 if err != nil { 569 return nil, b, err 570 } 571 572 var opener func() (afero.File, error) 573 if fi.IsDir() { 574 // Make sure metadata gets applied in Readdir. 575 opener = fs.realDirOpener(filename, root.Meta) 576 } else { 577 // Opens the real file directly. 578 opener = func() (afero.File, error) { 579 return fs.Fs.Open(filename) 580 } 581 } 582 583 return decorateFileInfo(fi, fs.Fs, opener, "", "", root.Meta), b, nil 584 } 585 586 func (fs *RootMappingFs) virtualDirOpener(name string) func() (afero.File, error) { 587 return func() (afero.File, error) { return &rootMappingFile{name: name, fs: fs}, nil } 588 } 589 590 func (fs *RootMappingFs) realDirOpener(name string, meta *FileMeta) func() (afero.File, error) { 591 return func() (afero.File, error) { 592 f, err := fs.Fs.Open(name) 593 if err != nil { 594 return nil, err 595 } 596 return &rootMappingFile{name: name, meta: meta, fs: fs, File: f}, nil 597 } 598 } 599 600 type rootMappingFile struct { 601 afero.File 602 fs *RootMappingFs 603 name string 604 meta *FileMeta 605 } 606 607 func (f *rootMappingFile) Close() error { 608 if f.File == nil { 609 return nil 610 } 611 return f.File.Close() 612 } 613 614 func (f *rootMappingFile) Name() string { 615 return f.name 616 } 617 618 func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) { 619 if f.File != nil { 620 621 fis, err := f.File.Readdir(count) 622 if err != nil { 623 return nil, err 624 } 625 626 var result []os.FileInfo 627 for _, fi := range fis { 628 fim := decorateFileInfo(fi, f.fs, nil, "", "", f.meta) 629 meta := fim.Meta() 630 if f.meta.InclusionFilter.Match(strings.TrimPrefix(meta.Filename, meta.SourceRoot), fim.IsDir()) { 631 result = append(result, fim) 632 } 633 } 634 return result, nil 635 } 636 637 return f.fs.collectDirEntries(f.name) 638 } 639 640 func (f *rootMappingFile) Readdirnames(count int) ([]string, error) { 641 dirs, err := f.Readdir(count) 642 if err != nil { 643 return nil, err 644 } 645 return fileInfosToNames(dirs), nil 646 }