github.com/kovansky/hugo@v0.92.3-0.20220224232819-63076e4ff19f/hugolib/hugo_sites.go (about) 1 // Copyright 2019 The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package hugolib 15 16 import ( 17 "context" 18 "io" 19 "path/filepath" 20 "sort" 21 "strings" 22 "sync" 23 "sync/atomic" 24 25 "github.com/gohugoio/hugo/hugofs/glob" 26 27 "github.com/fsnotify/fsnotify" 28 29 "github.com/gohugoio/hugo/identity" 30 31 radix "github.com/armon/go-radix" 32 33 "github.com/gohugoio/hugo/output" 34 "github.com/gohugoio/hugo/parser/metadecoders" 35 36 "github.com/gohugoio/hugo/common/para" 37 "github.com/gohugoio/hugo/hugofs" 38 "github.com/pkg/errors" 39 40 "github.com/gohugoio/hugo/source" 41 42 "github.com/bep/gitmap" 43 "github.com/gohugoio/hugo/config" 44 45 "github.com/gohugoio/hugo/publisher" 46 47 "github.com/gohugoio/hugo/common/herrors" 48 "github.com/gohugoio/hugo/common/loggers" 49 "github.com/gohugoio/hugo/deps" 50 "github.com/gohugoio/hugo/helpers" 51 "github.com/gohugoio/hugo/langs" 52 "github.com/gohugoio/hugo/lazy" 53 54 "github.com/gohugoio/hugo/langs/i18n" 55 "github.com/gohugoio/hugo/resources/page" 56 "github.com/gohugoio/hugo/resources/page/pagemeta" 57 "github.com/gohugoio/hugo/tpl" 58 "github.com/gohugoio/hugo/tpl/tplimpl" 59 ) 60 61 // HugoSites represents the sites to build. Each site represents a language. 62 type HugoSites struct { 63 Sites []*Site 64 65 multilingual *Multilingual 66 67 // Multihost is set if multilingual and baseURL set on the language level. 68 multihost bool 69 70 // If this is running in the dev server. 71 running bool 72 73 // Render output formats for all sites. 74 renderFormats output.Formats 75 76 *deps.Deps 77 78 gitInfo *gitInfo 79 codeownerInfo *codeownerInfo 80 81 // As loaded from the /data dirs 82 data map[string]interface{} 83 84 contentInit sync.Once 85 content *pageMaps 86 87 // Keeps track of bundle directories and symlinks to enable partial rebuilding. 88 ContentChanges *contentChangeMap 89 90 // File change events with filename stored in this map will be skipped. 91 skipRebuildForFilenamesMu sync.Mutex 92 skipRebuildForFilenames map[string]bool 93 94 init *hugoSitesInit 95 96 workers *para.Workers 97 numWorkers int 98 99 *fatalErrorHandler 100 *testCounters 101 } 102 103 // ShouldSkipFileChangeEvent allows skipping filesystem event early before 104 // the build is started. 105 func (h *HugoSites) ShouldSkipFileChangeEvent(ev fsnotify.Event) bool { 106 h.skipRebuildForFilenamesMu.Lock() 107 defer h.skipRebuildForFilenamesMu.Unlock() 108 return h.skipRebuildForFilenames[ev.Name] 109 } 110 111 func (h *HugoSites) getContentMaps() *pageMaps { 112 h.contentInit.Do(func() { 113 h.content = newPageMaps(h) 114 }) 115 return h.content 116 } 117 118 // Only used in tests. 119 type testCounters struct { 120 contentRenderCounter uint64 121 pageRenderCounter uint64 122 } 123 124 func (h *testCounters) IncrContentRender() { 125 if h == nil { 126 return 127 } 128 atomic.AddUint64(&h.contentRenderCounter, 1) 129 } 130 131 func (h *testCounters) IncrPageRender() { 132 if h == nil { 133 return 134 } 135 atomic.AddUint64(&h.pageRenderCounter, 1) 136 } 137 138 type fatalErrorHandler struct { 139 mu sync.Mutex 140 141 h *HugoSites 142 143 err error 144 145 done bool 146 donec chan bool // will be closed when done 147 } 148 149 // FatalError error is used in some rare situations where it does not make sense to 150 // continue processing, to abort as soon as possible and log the error. 151 func (f *fatalErrorHandler) FatalError(err error) { 152 f.mu.Lock() 153 defer f.mu.Unlock() 154 if !f.done { 155 f.done = true 156 close(f.donec) 157 } 158 f.err = err 159 } 160 161 func (f *fatalErrorHandler) getErr() error { 162 f.mu.Lock() 163 defer f.mu.Unlock() 164 return f.err 165 } 166 167 func (f *fatalErrorHandler) Done() <-chan bool { 168 return f.donec 169 } 170 171 type hugoSitesInit struct { 172 // Loads the data from all of the /data folders. 173 data *lazy.Init 174 175 // Performs late initialization (before render) of the templates. 176 layouts *lazy.Init 177 178 // Loads the Git info and CODEOWNERS for all the pages if enabled. 179 gitInfo *lazy.Init 180 181 // Maps page translations. 182 translations *lazy.Init 183 } 184 185 func (h *hugoSitesInit) Reset() { 186 h.data.Reset() 187 h.layouts.Reset() 188 h.gitInfo.Reset() 189 h.translations.Reset() 190 } 191 192 func (h *HugoSites) Data() map[string]interface{} { 193 if _, err := h.init.data.Do(); err != nil { 194 h.SendError(errors.Wrap(err, "failed to load data")) 195 return nil 196 } 197 return h.data 198 } 199 200 func (h *HugoSites) gitInfoForPage(p page.Page) (*gitmap.GitInfo, error) { 201 if _, err := h.init.gitInfo.Do(); err != nil { 202 return nil, err 203 } 204 205 if h.gitInfo == nil { 206 return nil, nil 207 } 208 209 return h.gitInfo.forPage(p), nil 210 } 211 212 func (h *HugoSites) codeownersForPage(p page.Page) ([]string, error) { 213 if _, err := h.init.gitInfo.Do(); err != nil { 214 return nil, err 215 } 216 217 if h.codeownerInfo == nil { 218 return nil, nil 219 } 220 221 return h.codeownerInfo.forPage(p), nil 222 } 223 224 func (h *HugoSites) siteInfos() page.Sites { 225 infos := make(page.Sites, len(h.Sites)) 226 for i, site := range h.Sites { 227 infos[i] = site.Info 228 } 229 return infos 230 } 231 232 func (h *HugoSites) pickOneAndLogTheRest(errors []error) error { 233 if len(errors) == 0 { 234 return nil 235 } 236 237 var i int 238 239 for j, err := range errors { 240 // If this is in server mode, we want to return an error to the client 241 // with a file context, if possible. 242 if herrors.UnwrapErrorWithFileContext(err) != nil { 243 i = j 244 break 245 } 246 } 247 248 // Log the rest, but add a threshold to avoid flooding the log. 249 const errLogThreshold = 5 250 251 for j, err := range errors { 252 if j == i || err == nil { 253 continue 254 } 255 256 if j >= errLogThreshold { 257 break 258 } 259 260 h.Log.Errorln(err) 261 } 262 263 return errors[i] 264 } 265 266 func (h *HugoSites) IsMultihost() bool { 267 return h != nil && h.multihost 268 } 269 270 // TODO(bep) consolidate 271 func (h *HugoSites) LanguageSet() map[string]int { 272 set := make(map[string]int) 273 for i, s := range h.Sites { 274 set[s.language.Lang] = i 275 } 276 return set 277 } 278 279 func (h *HugoSites) NumLogErrors() int { 280 if h == nil { 281 return 0 282 } 283 return int(h.Log.LogCounters().ErrorCounter.Count()) 284 } 285 286 func (h *HugoSites) PrintProcessingStats(w io.Writer) { 287 stats := make([]*helpers.ProcessingStats, len(h.Sites)) 288 for i := 0; i < len(h.Sites); i++ { 289 stats[i] = h.Sites[i].PathSpec.ProcessingStats 290 } 291 helpers.ProcessingStatsTable(w, stats...) 292 } 293 294 // GetContentPage finds a Page with content given the absolute filename. 295 // Returns nil if none found. 296 func (h *HugoSites) GetContentPage(filename string) page.Page { 297 var p page.Page 298 299 h.getContentMaps().walkBundles(func(b *contentNode) bool { 300 if b.p == nil || b.fi == nil { 301 return false 302 } 303 304 if b.fi.Meta().Filename == filename { 305 p = b.p 306 return true 307 } 308 309 return false 310 }) 311 312 return p 313 } 314 315 // NewHugoSites creates a new collection of sites given the input sites, building 316 // a language configuration based on those. 317 func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { 318 if cfg.Language != nil { 319 return nil, errors.New("Cannot provide Language in Cfg when sites are provided") 320 } 321 322 // Return error at the end. Make the caller decide if it's fatal or not. 323 var initErr error 324 325 langConfig, err := newMultiLingualFromSites(cfg.Cfg, sites...) 326 if err != nil { 327 return nil, errors.Wrap(err, "failed to create language config") 328 } 329 330 var contentChangeTracker *contentChangeMap 331 332 numWorkers := config.GetNumWorkerMultiplier() 333 if numWorkers > len(sites) { 334 numWorkers = len(sites) 335 } 336 var workers *para.Workers 337 if numWorkers > 1 { 338 workers = para.New(numWorkers) 339 } 340 341 h := &HugoSites{ 342 running: cfg.Running, 343 multilingual: langConfig, 344 multihost: cfg.Cfg.GetBool("multihost"), 345 Sites: sites, 346 workers: workers, 347 numWorkers: numWorkers, 348 skipRebuildForFilenames: make(map[string]bool), 349 init: &hugoSitesInit{ 350 data: lazy.New(), 351 layouts: lazy.New(), 352 gitInfo: lazy.New(), 353 translations: lazy.New(), 354 }, 355 } 356 357 h.fatalErrorHandler = &fatalErrorHandler{ 358 h: h, 359 donec: make(chan bool), 360 } 361 362 h.init.data.Add(func() (interface{}, error) { 363 err := h.loadData(h.PathSpec.BaseFs.Data.Dirs) 364 if err != nil { 365 return nil, errors.Wrap(err, "failed to load data") 366 } 367 return nil, nil 368 }) 369 370 h.init.layouts.Add(func() (interface{}, error) { 371 for _, s := range h.Sites { 372 if err := s.Tmpl().(tpl.TemplateManager).MarkReady(); err != nil { 373 return nil, err 374 } 375 } 376 return nil, nil 377 }) 378 379 h.init.translations.Add(func() (interface{}, error) { 380 if len(h.Sites) > 1 { 381 allTranslations := pagesToTranslationsMap(h.Sites) 382 assignTranslationsToPages(allTranslations, h.Sites) 383 } 384 385 return nil, nil 386 }) 387 388 h.init.gitInfo.Add(func() (interface{}, error) { 389 err := h.loadGitInfo() 390 if err != nil { 391 return nil, errors.Wrap(err, "failed to load Git info") 392 } 393 return nil, nil 394 }) 395 396 for _, s := range sites { 397 s.h = h 398 } 399 400 var l configLoader 401 if err := l.applyDeps(cfg, sites...); err != nil { 402 initErr = errors.Wrap(err, "add site dependencies") 403 } 404 405 h.Deps = sites[0].Deps 406 if h.Deps == nil { 407 return nil, initErr 408 } 409 410 // Only needed in server mode. 411 // TODO(bep) clean up the running vs watching terms 412 if cfg.Running { 413 contentChangeTracker = &contentChangeMap{ 414 pathSpec: h.PathSpec, 415 symContent: make(map[string]map[string]bool), 416 leafBundles: radix.New(), 417 branchBundles: make(map[string]bool), 418 } 419 h.ContentChanges = contentChangeTracker 420 } 421 422 return h, initErr 423 } 424 425 func (h *HugoSites) loadGitInfo() error { 426 if h.Cfg.GetBool("enableGitInfo") { 427 gi, err := newGitInfo(h.Cfg) 428 if err != nil { 429 h.Log.Errorln("Failed to read Git log:", err) 430 } else { 431 h.gitInfo = gi 432 } 433 434 co, err := newCodeowners(h.Cfg) 435 if err != nil { 436 h.Log.Errorln("Failed to read CODEOWNERS:", err) 437 } else { 438 h.codeownerInfo = co 439 } 440 } 441 return nil 442 } 443 444 func (l configLoader) applyDeps(cfg deps.DepsCfg, sites ...*Site) error { 445 if cfg.TemplateProvider == nil { 446 cfg.TemplateProvider = tplimpl.DefaultTemplateProvider 447 } 448 449 if cfg.TranslationProvider == nil { 450 cfg.TranslationProvider = i18n.NewTranslationProvider() 451 } 452 453 var ( 454 d *deps.Deps 455 err error 456 ) 457 458 for _, s := range sites { 459 if s.Deps != nil { 460 continue 461 } 462 463 onCreated := func(d *deps.Deps) error { 464 s.Deps = d 465 466 // Set up the main publishing chain. 467 pub, err := publisher.NewDestinationPublisher( 468 d.ResourceSpec, 469 s.outputFormatsConfig, 470 s.mediaTypesConfig, 471 ) 472 if err != nil { 473 return err 474 } 475 s.publisher = pub 476 477 if err := s.initializeSiteInfo(); err != nil { 478 return err 479 } 480 481 d.Site = s.Info 482 483 siteConfig, err := l.loadSiteConfig(s.language) 484 if err != nil { 485 return errors.Wrap(err, "load site config") 486 } 487 s.siteConfigConfig = siteConfig 488 489 pm := &pageMap{ 490 contentMap: newContentMap(contentMapConfig{ 491 lang: s.Lang(), 492 taxonomyConfig: s.siteCfg.taxonomiesConfig.Values(), 493 taxonomyDisabled: !s.isEnabled(page.KindTerm), 494 taxonomyTermDisabled: !s.isEnabled(page.KindTaxonomy), 495 pageDisabled: !s.isEnabled(page.KindPage), 496 }), 497 s: s, 498 } 499 500 s.PageCollections = newPageCollections(pm) 501 502 s.siteRefLinker, err = newSiteRefLinker(s.language, s) 503 return err 504 } 505 506 cfg.Language = s.language 507 cfg.MediaTypes = s.mediaTypesConfig 508 cfg.OutputFormats = s.outputFormatsConfig 509 510 if d == nil { 511 cfg.WithTemplate = s.withSiteTemplates(cfg.WithTemplate) 512 513 var err error 514 d, err = deps.New(cfg) 515 if err != nil { 516 return errors.Wrap(err, "create deps") 517 } 518 519 d.OutputFormatsConfig = s.outputFormatsConfig 520 521 if err := onCreated(d); err != nil { 522 return errors.Wrap(err, "on created") 523 } 524 525 if err = d.LoadResources(); err != nil { 526 return errors.Wrap(err, "load resources") 527 } 528 529 } else { 530 d, err = d.ForLanguage(cfg, onCreated) 531 if err != nil { 532 return err 533 } 534 d.OutputFormatsConfig = s.outputFormatsConfig 535 } 536 } 537 538 return nil 539 } 540 541 // NewHugoSites creates HugoSites from the given config. 542 func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { 543 if cfg.Logger == nil { 544 cfg.Logger = loggers.NewErrorLogger() 545 } 546 sites, err := createSitesFromConfig(cfg) 547 if err != nil { 548 return nil, errors.Wrap(err, "from config") 549 } 550 return newHugoSites(cfg, sites...) 551 } 552 553 func (s *Site) withSiteTemplates(withTemplates ...func(templ tpl.TemplateManager) error) func(templ tpl.TemplateManager) error { 554 return func(templ tpl.TemplateManager) error { 555 for _, wt := range withTemplates { 556 if wt == nil { 557 continue 558 } 559 if err := wt(templ); err != nil { 560 return err 561 } 562 } 563 564 return nil 565 } 566 } 567 568 func createSitesFromConfig(cfg deps.DepsCfg) ([]*Site, error) { 569 var sites []*Site 570 571 languages := getLanguages(cfg.Cfg) 572 573 for _, lang := range languages { 574 if lang.Disabled { 575 continue 576 } 577 var s *Site 578 var err error 579 cfg.Language = lang 580 s, err = newSite(cfg) 581 582 if err != nil { 583 return nil, err 584 } 585 586 sites = append(sites, s) 587 } 588 589 return sites, nil 590 } 591 592 // Reset resets the sites and template caches etc., making it ready for a full rebuild. 593 func (h *HugoSites) reset(config *BuildCfg) { 594 if config.ResetState { 595 for i, s := range h.Sites { 596 h.Sites[i] = s.reset() 597 if r, ok := s.Fs.Destination.(hugofs.Reseter); ok { 598 r.Reset() 599 } 600 } 601 } 602 603 h.fatalErrorHandler = &fatalErrorHandler{ 604 h: h, 605 donec: make(chan bool), 606 } 607 608 h.init.Reset() 609 } 610 611 // resetLogs resets the log counters etc. Used to do a new build on the same sites. 612 func (h *HugoSites) resetLogs() { 613 h.Log.Reset() 614 loggers.GlobalErrorCounter.Reset() 615 for _, s := range h.Sites { 616 s.Deps.Log.Reset() 617 s.Deps.LogDistinct.Reset() 618 } 619 } 620 621 func (h *HugoSites) withSite(fn func(s *Site) error) error { 622 if h.workers == nil { 623 for _, s := range h.Sites { 624 if err := fn(s); err != nil { 625 return err 626 } 627 } 628 return nil 629 } 630 631 g, _ := h.workers.Start(context.Background()) 632 for _, s := range h.Sites { 633 s := s 634 g.Run(func() error { 635 return fn(s) 636 }) 637 } 638 return g.Wait() 639 } 640 641 func (h *HugoSites) createSitesFromConfig(cfg config.Provider) error { 642 oldLangs, _ := h.Cfg.Get("languagesSorted").(langs.Languages) 643 644 l := configLoader{cfg: h.Cfg} 645 if err := l.loadLanguageSettings(oldLangs); err != nil { 646 return err 647 } 648 649 depsCfg := deps.DepsCfg{Fs: h.Fs, Cfg: l.cfg} 650 651 sites, err := createSitesFromConfig(depsCfg) 652 if err != nil { 653 return err 654 } 655 656 langConfig, err := newMultiLingualFromSites(depsCfg.Cfg, sites...) 657 if err != nil { 658 return err 659 } 660 661 h.Sites = sites 662 663 for _, s := range sites { 664 s.h = h 665 } 666 667 var cl configLoader 668 if err := cl.applyDeps(depsCfg, sites...); err != nil { 669 return err 670 } 671 672 h.Deps = sites[0].Deps 673 674 h.multilingual = langConfig 675 h.multihost = h.Deps.Cfg.GetBool("multihost") 676 677 return nil 678 } 679 680 func (h *HugoSites) toSiteInfos() []*SiteInfo { 681 infos := make([]*SiteInfo, len(h.Sites)) 682 for i, s := range h.Sites { 683 infos[i] = s.Info 684 } 685 return infos 686 } 687 688 // BuildCfg holds build options used to, as an example, skip the render step. 689 type BuildCfg struct { 690 // Reset site state before build. Use to force full rebuilds. 691 ResetState bool 692 // If set, we re-create the sites from the given configuration before a build. 693 // This is needed if new languages are added. 694 NewConfig config.Provider 695 // Skip rendering. Useful for testing. 696 SkipRender bool 697 // Use this to indicate what changed (for rebuilds). 698 whatChanged *whatChanged 699 700 // This is a partial re-render of some selected pages. This means 701 // we should skip most of the processing. 702 PartialReRender bool 703 704 // Set in server mode when the last build failed for some reason. 705 ErrRecovery bool 706 707 // Recently visited URLs. This is used for partial re-rendering. 708 RecentlyVisited map[string]bool 709 710 // Can be set to build only with a sub set of the content source. 711 ContentInclusionFilter *glob.FilenameFilter 712 713 // Set when the buildlock is already acquired (e.g. the archetype content builder). 714 NoBuildLock bool 715 716 testCounters *testCounters 717 } 718 719 // shouldRender is used in the Fast Render Mode to determine if we need to re-render 720 // a Page: If it is recently visited (the home pages will always be in this set) or changed. 721 // Note that a page does not have to have a content page / file. 722 // For regular builds, this will allways return true. 723 // TODO(bep) rename/work this. 724 func (cfg *BuildCfg) shouldRender(p *pageState) bool { 725 if p.forceRender { 726 return true 727 } 728 729 if len(cfg.RecentlyVisited) == 0 { 730 return true 731 } 732 733 if cfg.RecentlyVisited[p.RelPermalink()] { 734 return true 735 } 736 737 if cfg.whatChanged != nil && !p.File().IsZero() { 738 return cfg.whatChanged.files[p.File().Filename()] 739 } 740 741 return false 742 } 743 744 func (h *HugoSites) renderCrossSitesSitemap() error { 745 if !h.multilingual.enabled() || h.IsMultihost() { 746 return nil 747 } 748 749 sitemapEnabled := false 750 for _, s := range h.Sites { 751 if s.isEnabled(kindSitemap) { 752 sitemapEnabled = true 753 break 754 } 755 } 756 757 if !sitemapEnabled { 758 return nil 759 } 760 761 s := h.Sites[0] 762 763 templ := s.lookupLayouts("sitemapindex.xml", "_default/sitemapindex.xml", "_internal/_default/sitemapindex.xml") 764 765 return s.renderAndWriteXML(&s.PathSpec.ProcessingStats.Sitemaps, "sitemapindex", 766 s.siteCfg.sitemap.Filename, h.toSiteInfos(), templ) 767 } 768 769 func (h *HugoSites) renderCrossSitesRobotsTXT() error { 770 if h.multihost { 771 return nil 772 } 773 if !h.Cfg.GetBool("enableRobotsTXT") { 774 return nil 775 } 776 777 s := h.Sites[0] 778 779 p, err := newPageStandalone(&pageMeta{ 780 s: s, 781 kind: kindRobotsTXT, 782 urlPaths: pagemeta.URLPath{ 783 URL: "robots.txt", 784 }, 785 }, 786 output.RobotsTxtFormat) 787 if err != nil { 788 return err 789 } 790 791 if !p.render { 792 return nil 793 } 794 795 templ := s.lookupLayouts("robots.txt", "_default/robots.txt", "_internal/_default/robots.txt") 796 797 return s.renderAndWritePage(&s.PathSpec.ProcessingStats.Pages, "Robots Txt", "robots.txt", p, templ) 798 } 799 800 func (h *HugoSites) removePageByFilename(filename string) { 801 h.getContentMaps().withMaps(func(m *pageMap) error { 802 m.deleteBundleMatching(func(b *contentNode) bool { 803 if b.p == nil { 804 return false 805 } 806 807 if b.fi == nil { 808 return false 809 } 810 811 return b.fi.Meta().Filename == filename 812 }) 813 return nil 814 }) 815 } 816 817 func (h *HugoSites) createPageCollections() error { 818 allPages := newLazyPagesFactory(func() page.Pages { 819 var pages page.Pages 820 for _, s := range h.Sites { 821 pages = append(pages, s.Pages()...) 822 } 823 824 page.SortByDefault(pages) 825 826 return pages 827 }) 828 829 allRegularPages := newLazyPagesFactory(func() page.Pages { 830 return h.findPagesByKindIn(page.KindPage, allPages.get()) 831 }) 832 833 for _, s := range h.Sites { 834 s.PageCollections.allPages = allPages 835 s.PageCollections.allRegularPages = allRegularPages 836 } 837 838 return nil 839 } 840 841 func (s *Site) preparePagesForRender(isRenderingSite bool, idx int) error { 842 var err error 843 s.pageMap.withEveryBundlePage(func(p *pageState) bool { 844 if err = p.initOutputFormat(isRenderingSite, idx); err != nil { 845 return true 846 } 847 return false 848 }) 849 return nil 850 } 851 852 // Pages returns all pages for all sites. 853 func (h *HugoSites) Pages() page.Pages { 854 return h.Sites[0].AllPages() 855 } 856 857 func (h *HugoSites) loadData(fis []hugofs.FileMetaInfo) (err error) { 858 spec := source.NewSourceSpec(h.PathSpec, nil, nil) 859 860 h.data = make(map[string]interface{}) 861 for _, fi := range fis { 862 fileSystem := spec.NewFilesystemFromFileMetaInfo(fi) 863 files, err := fileSystem.Files() 864 if err != nil { 865 return err 866 } 867 for _, r := range files { 868 if err := h.handleDataFile(r); err != nil { 869 return err 870 } 871 } 872 } 873 874 return 875 } 876 877 func (h *HugoSites) handleDataFile(r source.File) error { 878 var current map[string]interface{} 879 880 f, err := r.FileInfo().Meta().Open() 881 if err != nil { 882 return errors.Wrapf(err, "data: failed to open %q:", r.LogicalName()) 883 } 884 defer f.Close() 885 886 // Crawl in data tree to insert data 887 current = h.data 888 keyParts := strings.Split(r.Dir(), helpers.FilePathSeparator) 889 890 for _, key := range keyParts { 891 if key != "" { 892 if _, ok := current[key]; !ok { 893 current[key] = make(map[string]interface{}) 894 } 895 current = current[key].(map[string]interface{}) 896 } 897 } 898 899 data, err := h.readData(r) 900 if err != nil { 901 return h.errWithFileContext(err, r) 902 } 903 904 if data == nil { 905 return nil 906 } 907 908 // filepath.Walk walks the files in lexical order, '/' comes before '.' 909 higherPrecedentData := current[r.BaseFileName()] 910 911 switch data.(type) { 912 case nil: 913 case map[string]interface{}: 914 915 switch higherPrecedentData.(type) { 916 case nil: 917 current[r.BaseFileName()] = data 918 case map[string]interface{}: 919 // merge maps: insert entries from data for keys that 920 // don't already exist in higherPrecedentData 921 higherPrecedentMap := higherPrecedentData.(map[string]interface{}) 922 for key, value := range data.(map[string]interface{}) { 923 if _, exists := higherPrecedentMap[key]; exists { 924 // this warning could happen if 925 // 1. A theme uses the same key; the main data folder wins 926 // 2. A sub folder uses the same key: the sub folder wins 927 // TODO(bep) figure out a way to detect 2) above and make that a WARN 928 h.Log.Infof("Data for key '%s' in path '%s' is overridden by higher precedence data already in the data tree", key, r.Path()) 929 } else { 930 higherPrecedentMap[key] = value 931 } 932 } 933 default: 934 // can't merge: higherPrecedentData is not a map 935 h.Log.Warnf("The %T data from '%s' overridden by "+ 936 "higher precedence %T data already in the data tree", data, r.Path(), higherPrecedentData) 937 } 938 939 case []interface{}: 940 if higherPrecedentData == nil { 941 current[r.BaseFileName()] = data 942 } else { 943 // we don't merge array data 944 h.Log.Warnf("The %T data from '%s' overridden by "+ 945 "higher precedence %T data already in the data tree", data, r.Path(), higherPrecedentData) 946 } 947 948 default: 949 h.Log.Errorf("unexpected data type %T in file %s", data, r.LogicalName()) 950 } 951 952 return nil 953 } 954 955 func (h *HugoSites) errWithFileContext(err error, f source.File) error { 956 fim, ok := f.FileInfo().(hugofs.FileMetaInfo) 957 if !ok { 958 return err 959 } 960 961 realFilename := fim.Meta().Filename 962 963 err, _ = herrors.WithFileContextForFile( 964 err, 965 realFilename, 966 realFilename, 967 h.SourceSpec.Fs.Source, 968 herrors.SimpleLineMatcher) 969 970 return err 971 } 972 973 func (h *HugoSites) readData(f source.File) (interface{}, error) { 974 file, err := f.FileInfo().Meta().Open() 975 if err != nil { 976 return nil, errors.Wrap(err, "readData: failed to open data file") 977 } 978 defer file.Close() 979 content := helpers.ReaderToBytes(file) 980 981 format := metadecoders.FormatFromString(f.Extension()) 982 return metadecoders.Default.Unmarshal(content, format) 983 } 984 985 func (h *HugoSites) findPagesByKindIn(kind string, inPages page.Pages) page.Pages { 986 return h.Sites[0].findPagesByKindIn(kind, inPages) 987 } 988 989 func (h *HugoSites) resetPageState() { 990 h.getContentMaps().walkBundles(func(n *contentNode) bool { 991 if n.p == nil { 992 return false 993 } 994 p := n.p 995 for _, po := range p.pageOutputs { 996 if po.cp == nil { 997 continue 998 } 999 po.cp.Reset() 1000 } 1001 1002 return false 1003 }) 1004 } 1005 1006 func (h *HugoSites) resetPageStateFromEvents(idset identity.Identities) { 1007 h.getContentMaps().walkBundles(func(n *contentNode) bool { 1008 if n.p == nil { 1009 return false 1010 } 1011 p := n.p 1012 OUTPUTS: 1013 for _, po := range p.pageOutputs { 1014 if po.cp == nil { 1015 continue 1016 } 1017 for id := range idset { 1018 if po.cp.dependencyTracker.Search(id) != nil { 1019 po.cp.Reset() 1020 continue OUTPUTS 1021 } 1022 } 1023 } 1024 1025 if p.shortcodeState == nil { 1026 return false 1027 } 1028 1029 for _, s := range p.shortcodeState.shortcodes { 1030 for _, templ := range s.templs { 1031 sid := templ.(identity.Manager) 1032 for id := range idset { 1033 if sid.Search(id) != nil { 1034 for _, po := range p.pageOutputs { 1035 if po.cp != nil { 1036 po.cp.Reset() 1037 } 1038 } 1039 return false 1040 } 1041 } 1042 } 1043 } 1044 return false 1045 }) 1046 } 1047 1048 // Used in partial reloading to determine if the change is in a bundle. 1049 type contentChangeMap struct { 1050 mu sync.RWMutex 1051 1052 // Holds directories with leaf bundles. 1053 leafBundles *radix.Tree 1054 1055 // Holds directories with branch bundles. 1056 branchBundles map[string]bool 1057 1058 pathSpec *helpers.PathSpec 1059 1060 // Hugo supports symlinked content (both directories and files). This 1061 // can lead to situations where the same file can be referenced from several 1062 // locations in /content -- which is really cool, but also means we have to 1063 // go an extra mile to handle changes. 1064 // This map is only used in watch mode. 1065 // It maps either file to files or the real dir to a set of content directories 1066 // where it is in use. 1067 symContentMu sync.Mutex 1068 symContent map[string]map[string]bool 1069 } 1070 1071 func (m *contentChangeMap) add(dirname string, tp bundleDirType) { 1072 m.mu.Lock() 1073 if !strings.HasSuffix(dirname, helpers.FilePathSeparator) { 1074 dirname += helpers.FilePathSeparator 1075 } 1076 switch tp { 1077 case bundleBranch: 1078 m.branchBundles[dirname] = true 1079 case bundleLeaf: 1080 m.leafBundles.Insert(dirname, true) 1081 default: 1082 m.mu.Unlock() 1083 panic("invalid bundle type") 1084 } 1085 m.mu.Unlock() 1086 } 1087 1088 func (m *contentChangeMap) resolveAndRemove(filename string) (string, bundleDirType) { 1089 m.mu.RLock() 1090 defer m.mu.RUnlock() 1091 1092 // Bundles share resources, so we need to start from the virtual root. 1093 relFilename := m.pathSpec.RelContentDir(filename) 1094 dir, name := filepath.Split(relFilename) 1095 if !strings.HasSuffix(dir, helpers.FilePathSeparator) { 1096 dir += helpers.FilePathSeparator 1097 } 1098 1099 if _, found := m.branchBundles[dir]; found { 1100 delete(m.branchBundles, dir) 1101 return dir, bundleBranch 1102 } 1103 1104 if key, _, found := m.leafBundles.LongestPrefix(dir); found { 1105 m.leafBundles.Delete(key) 1106 dir = string(key) 1107 return dir, bundleLeaf 1108 } 1109 1110 fileTp, isContent := classifyBundledFile(name) 1111 if isContent && fileTp != bundleNot { 1112 // A new bundle. 1113 return dir, fileTp 1114 } 1115 1116 return dir, bundleNot 1117 } 1118 1119 func (m *contentChangeMap) addSymbolicLinkMapping(fim hugofs.FileMetaInfo) { 1120 meta := fim.Meta() 1121 if !meta.IsSymlink { 1122 return 1123 } 1124 m.symContentMu.Lock() 1125 1126 from, to := meta.Filename, meta.OriginalFilename 1127 if fim.IsDir() { 1128 if !strings.HasSuffix(from, helpers.FilePathSeparator) { 1129 from += helpers.FilePathSeparator 1130 } 1131 } 1132 1133 mm, found := m.symContent[from] 1134 1135 if !found { 1136 mm = make(map[string]bool) 1137 m.symContent[from] = mm 1138 } 1139 mm[to] = true 1140 m.symContentMu.Unlock() 1141 } 1142 1143 func (m *contentChangeMap) GetSymbolicLinkMappings(dir string) []string { 1144 mm, found := m.symContent[dir] 1145 if !found { 1146 return nil 1147 } 1148 dirs := make([]string, len(mm)) 1149 i := 0 1150 for dir := range mm { 1151 dirs[i] = dir 1152 i++ 1153 } 1154 1155 sort.Strings(dirs) 1156 1157 return dirs 1158 }