github.com/olliephillips/hugo@v0.42.2/hugolib/filesystems/basefs.go (about) 1 // Copyright 2018 The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 // Package filesystems provides the fine grained file systems used by Hugo. These 15 // are typically virtual filesystems that are composites of project and theme content. 16 package filesystems 17 18 import ( 19 "errors" 20 "io" 21 "os" 22 "path/filepath" 23 "strings" 24 25 "github.com/gohugoio/hugo/config" 26 27 "github.com/gohugoio/hugo/hugofs" 28 29 "fmt" 30 31 "github.com/gohugoio/hugo/common/types" 32 "github.com/gohugoio/hugo/hugolib/paths" 33 "github.com/gohugoio/hugo/langs" 34 "github.com/spf13/afero" 35 ) 36 37 // When we create a virtual filesystem with data and i18n bundles for the project and the themes, 38 // this is the name of the project's virtual root. It got it's funky name to make sure 39 // (or very unlikely) that it collides with a theme name. 40 const projectVirtualFolder = "__h__project" 41 42 var filePathSeparator = string(filepath.Separator) 43 44 // BaseFs contains the core base filesystems used by Hugo. The name "base" is used 45 // to underline that even if they can be composites, they all have a base path set to a specific 46 // resource folder, e.g "/my-project/content". So, no absolute filenames needed. 47 type BaseFs struct { 48 // TODO(bep) make this go away 49 AbsContentDirs []types.KeyValueStr 50 51 // The filesystem used to capture content. This can be a composite and 52 // language aware file system. 53 ContentFs afero.Fs 54 55 // SourceFilesystems contains the different source file systems. 56 *SourceFilesystems 57 58 // The filesystem used to store resources (processed images etc.). 59 // This usually maps to /my-project/resources. 60 ResourcesFs afero.Fs 61 62 // The filesystem used to publish the rendered site. 63 // This usually maps to /my-project/public. 64 PublishFs afero.Fs 65 66 themeFs afero.Fs 67 68 // TODO(bep) improve the "theme interaction" 69 AbsThemeDirs []string 70 } 71 72 // RelContentDir tries to create a path relative to the content root from 73 // the given filename. The return value is the path and language code. 74 func (b *BaseFs) RelContentDir(filename string) (string, string) { 75 for _, dir := range b.AbsContentDirs { 76 if strings.HasPrefix(filename, dir.Value) { 77 rel := strings.TrimPrefix(filename, dir.Value) 78 return strings.TrimPrefix(rel, filePathSeparator), dir.Key 79 } 80 } 81 // Either not a content dir or already relative. 82 return filename, "" 83 } 84 85 // IsContent returns whether the given filename is in the content filesystem. 86 func (b *BaseFs) IsContent(filename string) bool { 87 for _, dir := range b.AbsContentDirs { 88 if strings.HasPrefix(filename, dir.Value) { 89 return true 90 } 91 } 92 return false 93 } 94 95 // SourceFilesystems contains the different source file systems. These can be 96 // composite file systems (theme and project etc.), and they have all root 97 // set to the source type the provides: data, i18n, static, layouts. 98 type SourceFilesystems struct { 99 Data *SourceFilesystem 100 I18n *SourceFilesystem 101 Layouts *SourceFilesystem 102 Archetypes *SourceFilesystem 103 104 // When in multihost we have one static filesystem per language. The sync 105 // static files is currently done outside of the Hugo build (where there is 106 // a concept of a site per language). 107 // When in non-multihost mode there will be one entry in this map with a blank key. 108 Static map[string]*SourceFilesystem 109 } 110 111 // A SourceFilesystem holds the filesystem for a given source type in Hugo (data, 112 // i18n, layouts, static) and additional metadata to be able to use that filesystem 113 // in server mode. 114 type SourceFilesystem struct { 115 Fs afero.Fs 116 117 Dirnames []string 118 119 // When syncing a source folder to the target (e.g. /public), this may 120 // be set to publish into a subfolder. This is used for static syncing 121 // in multihost mode. 122 PublishFolder string 123 } 124 125 // IsStatic returns true if the given filename is a member of one of the static 126 // filesystems. 127 func (s SourceFilesystems) IsStatic(filename string) bool { 128 for _, staticFs := range s.Static { 129 if staticFs.Contains(filename) { 130 return true 131 } 132 } 133 return false 134 } 135 136 // IsLayout returns true if the given filename is a member of the layouts filesystem. 137 func (s SourceFilesystems) IsLayout(filename string) bool { 138 return s.Layouts.Contains(filename) 139 } 140 141 // IsData returns true if the given filename is a member of the data filesystem. 142 func (s SourceFilesystems) IsData(filename string) bool { 143 return s.Data.Contains(filename) 144 } 145 146 // IsI18n returns true if the given filename is a member of the i18n filesystem. 147 func (s SourceFilesystems) IsI18n(filename string) bool { 148 return s.I18n.Contains(filename) 149 } 150 151 // MakeStaticPathRelative makes an absolute static filename into a relative one. 152 // It will return an empty string if the filename is not a member of a static filesystem. 153 func (s SourceFilesystems) MakeStaticPathRelative(filename string) string { 154 for _, staticFs := range s.Static { 155 rel := staticFs.MakePathRelative(filename) 156 if rel != "" { 157 return rel 158 } 159 } 160 return "" 161 } 162 163 // MakePathRelative creates a relative path from the given filename. 164 // It will return an empty string if the filename is not a member of this filesystem. 165 func (d *SourceFilesystem) MakePathRelative(filename string) string { 166 for _, currentPath := range d.Dirnames { 167 if strings.HasPrefix(filename, currentPath) { 168 return strings.TrimPrefix(filename, currentPath) 169 } 170 } 171 return "" 172 } 173 174 // Contains returns whether the given filename is a member of the current filesystem. 175 func (d *SourceFilesystem) Contains(filename string) bool { 176 for _, dir := range d.Dirnames { 177 if strings.HasPrefix(filename, dir) { 178 return true 179 } 180 } 181 return false 182 } 183 184 // WithBaseFs allows reuse of some potentially expensive to create parts that remain 185 // the same across sites/languages. 186 func WithBaseFs(b *BaseFs) func(*BaseFs) error { 187 return func(bb *BaseFs) error { 188 bb.themeFs = b.themeFs 189 bb.AbsThemeDirs = b.AbsThemeDirs 190 return nil 191 } 192 } 193 194 // NewBase builds the filesystems used by Hugo given the paths and options provided.NewBase 195 func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) { 196 fs := p.Fs 197 198 resourcesFs := afero.NewBasePathFs(fs.Source, p.AbsResourcesDir) 199 publishFs := afero.NewBasePathFs(fs.Destination, p.AbsPublishDir) 200 201 contentFs, absContentDirs, err := createContentFs(fs.Source, p.WorkingDir, p.DefaultContentLanguage, p.Languages) 202 if err != nil { 203 return nil, err 204 } 205 206 // Make sure we don't have any overlapping content dirs. That will never work. 207 for i, d1 := range absContentDirs { 208 for j, d2 := range absContentDirs { 209 if i == j { 210 continue 211 } 212 if strings.HasPrefix(d1.Value, d2.Value) || strings.HasPrefix(d2.Value, d1.Value) { 213 return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2) 214 } 215 } 216 } 217 218 b := &BaseFs{ 219 AbsContentDirs: absContentDirs, 220 ContentFs: contentFs, 221 ResourcesFs: resourcesFs, 222 PublishFs: publishFs, 223 } 224 225 for _, opt := range options { 226 if err := opt(b); err != nil { 227 return nil, err 228 } 229 } 230 231 builder := newSourceFilesystemsBuilder(p, b) 232 sourceFilesystems, err := builder.Build() 233 if err != nil { 234 return nil, err 235 } 236 237 b.SourceFilesystems = sourceFilesystems 238 b.themeFs = builder.themeFs 239 b.AbsThemeDirs = builder.absThemeDirs 240 241 return b, nil 242 } 243 244 type sourceFilesystemsBuilder struct { 245 p *paths.Paths 246 result *SourceFilesystems 247 themeFs afero.Fs 248 hasTheme bool 249 absThemeDirs []string 250 } 251 252 func newSourceFilesystemsBuilder(p *paths.Paths, b *BaseFs) *sourceFilesystemsBuilder { 253 return &sourceFilesystemsBuilder{p: p, themeFs: b.themeFs, absThemeDirs: b.AbsThemeDirs, result: &SourceFilesystems{}} 254 } 255 256 func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) { 257 if b.themeFs == nil && b.p.ThemeSet() { 258 themeFs, absThemeDirs, err := createThemesOverlayFs(b.p) 259 if err != nil { 260 return nil, err 261 } 262 if themeFs == nil { 263 panic("createThemesFs returned nil") 264 } 265 b.themeFs = themeFs 266 b.absThemeDirs = absThemeDirs 267 268 } 269 270 b.hasTheme = len(b.absThemeDirs) > 0 271 272 sfs, err := b.createRootMappingFs("dataDir", "data") 273 if err != nil { 274 return nil, err 275 } 276 b.result.Data = sfs 277 278 sfs, err = b.createRootMappingFs("i18nDir", "i18n") 279 if err != nil { 280 return nil, err 281 } 282 b.result.I18n = sfs 283 284 sfs, err = b.createFs("layoutDir", "layouts") 285 if err != nil { 286 return nil, err 287 } 288 b.result.Layouts = sfs 289 290 sfs, err = b.createFs("archetypeDir", "archetypes") 291 if err != nil { 292 return nil, err 293 } 294 b.result.Archetypes = sfs 295 296 err = b.createStaticFs() 297 if err != nil { 298 return nil, err 299 } 300 301 return b.result, nil 302 } 303 304 func (b *sourceFilesystemsBuilder) createFs(dirKey, themeFolder string) (*SourceFilesystem, error) { 305 s := &SourceFilesystem{} 306 dir := b.p.Cfg.GetString(dirKey) 307 if dir == "" { 308 return s, fmt.Errorf("config %q not set", dirKey) 309 } 310 311 var fs afero.Fs 312 313 absDir := b.p.AbsPathify(dir) 314 if b.existsInSource(absDir) { 315 fs = afero.NewBasePathFs(b.p.Fs.Source, absDir) 316 s.Dirnames = []string{absDir} 317 } 318 319 if b.hasTheme { 320 themeFolderFs := afero.NewBasePathFs(b.themeFs, themeFolder) 321 if fs == nil { 322 fs = themeFolderFs 323 } else { 324 fs = afero.NewCopyOnWriteFs(themeFolderFs, fs) 325 } 326 327 for _, absThemeDir := range b.absThemeDirs { 328 absThemeFolderDir := filepath.Join(absThemeDir, themeFolder) 329 if b.existsInSource(absThemeFolderDir) { 330 s.Dirnames = append(s.Dirnames, absThemeFolderDir) 331 } 332 } 333 } 334 335 if fs == nil { 336 s.Fs = hugofs.NoOpFs 337 } else { 338 s.Fs = afero.NewReadOnlyFs(fs) 339 } 340 341 return s, nil 342 } 343 344 // Used for data, i18n -- we cannot use overlay filsesystems for those, but we need 345 // to keep a strict order. 346 func (b *sourceFilesystemsBuilder) createRootMappingFs(dirKey, themeFolder string) (*SourceFilesystem, error) { 347 s := &SourceFilesystem{} 348 349 projectDir := b.p.Cfg.GetString(dirKey) 350 if projectDir == "" { 351 return nil, fmt.Errorf("config %q not set", dirKey) 352 } 353 354 var fromTo []string 355 to := b.p.AbsPathify(projectDir) 356 357 if b.existsInSource(to) { 358 s.Dirnames = []string{to} 359 fromTo = []string{projectVirtualFolder, to} 360 } 361 362 for _, theme := range b.p.AllThemes { 363 to := b.p.AbsPathify(filepath.Join(b.p.ThemesDir, theme.Name, themeFolder)) 364 if b.existsInSource(to) { 365 s.Dirnames = append(s.Dirnames, to) 366 from := theme 367 fromTo = append(fromTo, from.Name, to) 368 } 369 } 370 371 if len(fromTo) == 0 { 372 s.Fs = hugofs.NoOpFs 373 return s, nil 374 } 375 376 fs, err := hugofs.NewRootMappingFs(b.p.Fs.Source, fromTo...) 377 if err != nil { 378 return nil, err 379 } 380 381 s.Fs = afero.NewReadOnlyFs(fs) 382 383 return s, nil 384 385 } 386 387 func (b *sourceFilesystemsBuilder) existsInSource(abspath string) bool { 388 exists, _ := afero.Exists(b.p.Fs.Source, abspath) 389 return exists 390 } 391 392 func (b *sourceFilesystemsBuilder) createStaticFs() error { 393 isMultihost := b.p.Cfg.GetBool("multihost") 394 ms := make(map[string]*SourceFilesystem) 395 b.result.Static = ms 396 397 if isMultihost { 398 for _, l := range b.p.Languages { 399 s := &SourceFilesystem{PublishFolder: l.Lang} 400 staticDirs := removeDuplicatesKeepRight(getStaticDirs(l)) 401 if len(staticDirs) == 0 { 402 continue 403 } 404 405 for _, dir := range staticDirs { 406 absDir := b.p.AbsPathify(dir) 407 if !b.existsInSource(absDir) { 408 continue 409 } 410 411 s.Dirnames = append(s.Dirnames, absDir) 412 } 413 414 fs, err := createOverlayFs(b.p.Fs.Source, s.Dirnames) 415 if err != nil { 416 return err 417 } 418 419 s.Fs = fs 420 ms[l.Lang] = s 421 422 } 423 424 return nil 425 } 426 427 s := &SourceFilesystem{} 428 var staticDirs []string 429 430 for _, l := range b.p.Languages { 431 staticDirs = append(staticDirs, getStaticDirs(l)...) 432 } 433 434 staticDirs = removeDuplicatesKeepRight(staticDirs) 435 if len(staticDirs) == 0 { 436 return nil 437 } 438 439 for _, dir := range staticDirs { 440 absDir := b.p.AbsPathify(dir) 441 if !b.existsInSource(absDir) { 442 continue 443 } 444 s.Dirnames = append(s.Dirnames, absDir) 445 } 446 447 fs, err := createOverlayFs(b.p.Fs.Source, s.Dirnames) 448 if err != nil { 449 return err 450 } 451 452 if b.hasTheme { 453 themeFolder := "static" 454 fs = afero.NewCopyOnWriteFs(afero.NewBasePathFs(b.themeFs, themeFolder), fs) 455 for _, absThemeDir := range b.absThemeDirs { 456 s.Dirnames = append(s.Dirnames, filepath.Join(absThemeDir, themeFolder)) 457 } 458 } 459 460 s.Fs = fs 461 ms[""] = s 462 463 return nil 464 } 465 466 func getStaticDirs(cfg config.Provider) []string { 467 var staticDirs []string 468 for i := -1; i <= 10; i++ { 469 staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...) 470 } 471 return staticDirs 472 } 473 474 func getStringOrStringSlice(cfg config.Provider, key string, id int) []string { 475 476 if id >= 0 { 477 key = fmt.Sprintf("%s%d", key, id) 478 } 479 480 return config.GetStringSlicePreserveString(cfg, key) 481 482 } 483 484 func createContentFs(fs afero.Fs, 485 workingDir, 486 defaultContentLanguage string, 487 languages langs.Languages) (afero.Fs, []types.KeyValueStr, error) { 488 489 var contentLanguages langs.Languages 490 var contentDirSeen = make(map[string]bool) 491 languageSet := make(map[string]bool) 492 493 // The default content language needs to be first. 494 for _, language := range languages { 495 if language.Lang == defaultContentLanguage { 496 contentLanguages = append(contentLanguages, language) 497 contentDirSeen[language.ContentDir] = true 498 } 499 languageSet[language.Lang] = true 500 } 501 502 for _, language := range languages { 503 if contentDirSeen[language.ContentDir] { 504 continue 505 } 506 if language.ContentDir == "" { 507 language.ContentDir = defaultContentLanguage 508 } 509 contentDirSeen[language.ContentDir] = true 510 contentLanguages = append(contentLanguages, language) 511 512 } 513 514 var absContentDirs []types.KeyValueStr 515 516 fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs) 517 return fs, absContentDirs, err 518 519 } 520 521 func createContentOverlayFs(source afero.Fs, 522 workingDir string, 523 languages langs.Languages, 524 languageSet map[string]bool, 525 absContentDirs *[]types.KeyValueStr) (afero.Fs, error) { 526 if len(languages) == 0 { 527 return source, nil 528 } 529 530 language := languages[0] 531 532 contentDir := language.ContentDir 533 if contentDir == "" { 534 panic("missing contentDir") 535 } 536 537 absContentDir := paths.AbsPathify(workingDir, language.ContentDir) 538 if !strings.HasSuffix(absContentDir, paths.FilePathSeparator) { 539 absContentDir += paths.FilePathSeparator 540 } 541 542 // If root, remove the second '/' 543 if absContentDir == "//" { 544 absContentDir = paths.FilePathSeparator 545 } 546 547 if len(absContentDir) < 6 { 548 return nil, fmt.Errorf("invalid content dir %q: Path is too short", absContentDir) 549 } 550 551 *absContentDirs = append(*absContentDirs, types.KeyValueStr{Key: language.Lang, Value: absContentDir}) 552 553 overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir)) 554 if len(languages) == 1 { 555 return overlay, nil 556 } 557 558 base, err := createContentOverlayFs(source, workingDir, languages[1:], languageSet, absContentDirs) 559 if err != nil { 560 return nil, err 561 } 562 563 return hugofs.NewLanguageCompositeFs(base, overlay), nil 564 565 } 566 567 func createThemesOverlayFs(p *paths.Paths) (afero.Fs, []string, error) { 568 569 themes := p.AllThemes 570 571 if len(themes) == 0 { 572 panic("AllThemes not set") 573 } 574 575 themesDir := p.AbsPathify(p.ThemesDir) 576 if themesDir == "" { 577 return nil, nil, errors.New("no themes dir set") 578 } 579 580 absPaths := make([]string, len(themes)) 581 582 // The themes are ordered from left to right. We need to revert it to get the 583 // overlay logic below working as expected. 584 for i := 0; i < len(themes); i++ { 585 absPaths[i] = filepath.Join(themesDir, themes[len(themes)-1-i].Name) 586 } 587 588 fs, err := createOverlayFs(p.Fs.Source, absPaths) 589 590 return fs, absPaths, err 591 592 } 593 594 func createOverlayFs(source afero.Fs, absPaths []string) (afero.Fs, error) { 595 if len(absPaths) == 0 { 596 return hugofs.NoOpFs, nil 597 } 598 599 if len(absPaths) == 1 { 600 return afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0])), nil 601 } 602 603 base := afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0])) 604 overlay, err := createOverlayFs(source, absPaths[1:]) 605 if err != nil { 606 return nil, err 607 } 608 609 return afero.NewCopyOnWriteFs(base, overlay), nil 610 } 611 612 func removeDuplicatesKeepRight(in []string) []string { 613 seen := make(map[string]bool) 614 var out []string 615 for i := len(in) - 1; i >= 0; i-- { 616 v := in[i] 617 if seen[v] { 618 continue 619 } 620 out = append([]string{v}, out...) 621 seen[v] = true 622 } 623 624 return out 625 } 626 627 func printFs(fs afero.Fs, path string, w io.Writer) { 628 if fs == nil { 629 return 630 } 631 afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error { 632 if info != nil && !info.IsDir() { 633 s := path 634 if lang, ok := info.(hugofs.LanguageAnnouncer); ok { 635 s = s + "\tLANG: " + lang.Lang() 636 } 637 if fp, ok := info.(hugofs.FilePather); ok { 638 s = s + "\tRF: " + fp.Filename() + "\tBP: " + fp.BaseDir() 639 } 640 fmt.Fprintln(w, " ", s) 641 } 642 return nil 643 }) 644 }