github.com/graemephi/kahugo@v0.62.3-0.20211121071557-d78c0423784d/hugolib/hugo_sites.go (about) 1 // Copyright 2019 The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package hugolib 15 16 import ( 17 "context" 18 "io" 19 "path/filepath" 20 "sort" 21 "strings" 22 "sync" 23 "sync/atomic" 24 25 "github.com/gohugoio/hugo/hugofs/glob" 26 27 "github.com/fsnotify/fsnotify" 28 29 "github.com/gohugoio/hugo/identity" 30 31 radix "github.com/armon/go-radix" 32 33 "github.com/gohugoio/hugo/output" 34 "github.com/gohugoio/hugo/parser/metadecoders" 35 36 "github.com/gohugoio/hugo/common/para" 37 "github.com/gohugoio/hugo/hugofs" 38 "github.com/pkg/errors" 39 40 "github.com/gohugoio/hugo/source" 41 42 "github.com/bep/gitmap" 43 "github.com/gohugoio/hugo/config" 44 45 "github.com/gohugoio/hugo/publisher" 46 47 "github.com/gohugoio/hugo/common/herrors" 48 "github.com/gohugoio/hugo/common/loggers" 49 "github.com/gohugoio/hugo/deps" 50 "github.com/gohugoio/hugo/helpers" 51 "github.com/gohugoio/hugo/langs" 52 "github.com/gohugoio/hugo/lazy" 53 54 "github.com/gohugoio/hugo/langs/i18n" 55 "github.com/gohugoio/hugo/resources/page" 56 "github.com/gohugoio/hugo/resources/page/pagemeta" 57 "github.com/gohugoio/hugo/tpl" 58 "github.com/gohugoio/hugo/tpl/tplimpl" 59 ) 60 61 // HugoSites represents the sites to build. Each site represents a language. 62 type HugoSites struct { 63 Sites []*Site 64 65 multilingual *Multilingual 66 67 // Multihost is set if multilingual and baseURL set on the language level. 68 multihost bool 69 70 // If this is running in the dev server. 71 running bool 72 73 // Render output formats for all sites. 74 renderFormats output.Formats 75 76 *deps.Deps 77 78 gitInfo *gitInfo 79 80 // As loaded from the /data dirs 81 data map[string]interface{} 82 83 contentInit sync.Once 84 content *pageMaps 85 86 // Keeps track of bundle directories and symlinks to enable partial rebuilding. 87 ContentChanges *contentChangeMap 88 89 // File change events with filename stored in this map will be skipped. 90 skipRebuildForFilenamesMu sync.Mutex 91 skipRebuildForFilenames map[string]bool 92 93 init *hugoSitesInit 94 95 workers *para.Workers 96 numWorkers int 97 98 *fatalErrorHandler 99 *testCounters 100 } 101 102 // ShouldSkipFileChangeEvent allows skipping filesystem event early before 103 // the build is started. 104 func (h *HugoSites) ShouldSkipFileChangeEvent(ev fsnotify.Event) bool { 105 h.skipRebuildForFilenamesMu.Lock() 106 defer h.skipRebuildForFilenamesMu.Unlock() 107 return h.skipRebuildForFilenames[ev.Name] 108 } 109 110 func (h *HugoSites) getContentMaps() *pageMaps { 111 h.contentInit.Do(func() { 112 h.content = newPageMaps(h) 113 }) 114 return h.content 115 } 116 117 // Only used in tests. 118 type testCounters struct { 119 contentRenderCounter uint64 120 } 121 122 func (h *testCounters) IncrContentRender() { 123 if h == nil { 124 return 125 } 126 atomic.AddUint64(&h.contentRenderCounter, 1) 127 } 128 129 type fatalErrorHandler struct { 130 mu sync.Mutex 131 132 h *HugoSites 133 134 err error 135 136 done bool 137 donec chan bool // will be closed when done 138 } 139 140 // FatalError error is used in some rare situations where it does not make sense to 141 // continue processing, to abort as soon as possible and log the error. 142 func (f *fatalErrorHandler) FatalError(err error) { 143 f.mu.Lock() 144 defer f.mu.Unlock() 145 if !f.done { 146 f.done = true 147 close(f.donec) 148 } 149 f.err = err 150 } 151 152 func (f *fatalErrorHandler) getErr() error { 153 f.mu.Lock() 154 defer f.mu.Unlock() 155 return f.err 156 } 157 158 func (f *fatalErrorHandler) Done() <-chan bool { 159 return f.donec 160 } 161 162 type hugoSitesInit struct { 163 // Loads the data from all of the /data folders. 164 data *lazy.Init 165 166 // Performs late initialization (before render) of the templates. 167 layouts *lazy.Init 168 169 // Loads the Git info for all the pages if enabled. 170 gitInfo *lazy.Init 171 172 // Maps page translations. 173 translations *lazy.Init 174 } 175 176 func (h *hugoSitesInit) Reset() { 177 h.data.Reset() 178 h.layouts.Reset() 179 h.gitInfo.Reset() 180 h.translations.Reset() 181 } 182 183 func (h *HugoSites) Data() map[string]interface{} { 184 if _, err := h.init.data.Do(); err != nil { 185 h.SendError(errors.Wrap(err, "failed to load data")) 186 return nil 187 } 188 return h.data 189 } 190 191 func (h *HugoSites) gitInfoForPage(p page.Page) (*gitmap.GitInfo, error) { 192 if _, err := h.init.gitInfo.Do(); err != nil { 193 return nil, err 194 } 195 196 if h.gitInfo == nil { 197 return nil, nil 198 } 199 200 return h.gitInfo.forPage(p), nil 201 } 202 203 func (h *HugoSites) siteInfos() page.Sites { 204 infos := make(page.Sites, len(h.Sites)) 205 for i, site := range h.Sites { 206 infos[i] = site.Info 207 } 208 return infos 209 } 210 211 func (h *HugoSites) pickOneAndLogTheRest(errors []error) error { 212 if len(errors) == 0 { 213 return nil 214 } 215 216 var i int 217 218 for j, err := range errors { 219 // If this is in server mode, we want to return an error to the client 220 // with a file context, if possible. 221 if herrors.UnwrapErrorWithFileContext(err) != nil { 222 i = j 223 break 224 } 225 } 226 227 // Log the rest, but add a threshold to avoid flooding the log. 228 const errLogThreshold = 5 229 230 for j, err := range errors { 231 if j == i || err == nil { 232 continue 233 } 234 235 if j >= errLogThreshold { 236 break 237 } 238 239 h.Log.Errorln(err) 240 } 241 242 return errors[i] 243 } 244 245 func (h *HugoSites) IsMultihost() bool { 246 return h != nil && h.multihost 247 } 248 249 // TODO(bep) consolidate 250 func (h *HugoSites) LanguageSet() map[string]int { 251 set := make(map[string]int) 252 for i, s := range h.Sites { 253 set[s.language.Lang] = i 254 } 255 return set 256 } 257 258 func (h *HugoSites) NumLogErrors() int { 259 if h == nil { 260 return 0 261 } 262 return int(h.Log.LogCounters().ErrorCounter.Count()) 263 } 264 265 func (h *HugoSites) PrintProcessingStats(w io.Writer) { 266 stats := make([]*helpers.ProcessingStats, len(h.Sites)) 267 for i := 0; i < len(h.Sites); i++ { 268 stats[i] = h.Sites[i].PathSpec.ProcessingStats 269 } 270 helpers.ProcessingStatsTable(w, stats...) 271 } 272 273 // GetContentPage finds a Page with content given the absolute filename. 274 // Returns nil if none found. 275 func (h *HugoSites) GetContentPage(filename string) page.Page { 276 var p page.Page 277 278 h.getContentMaps().walkBundles(func(b *contentNode) bool { 279 if b.p == nil || b.fi == nil { 280 return false 281 } 282 283 if b.fi.Meta().Filename == filename { 284 p = b.p 285 return true 286 } 287 288 return false 289 }) 290 291 return p 292 } 293 294 // NewHugoSites creates a new collection of sites given the input sites, building 295 // a language configuration based on those. 296 func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { 297 if cfg.Language != nil { 298 return nil, errors.New("Cannot provide Language in Cfg when sites are provided") 299 } 300 301 // Return error at the end. Make the caller decide if it's fatal or not. 302 var initErr error 303 304 langConfig, err := newMultiLingualFromSites(cfg.Cfg, sites...) 305 if err != nil { 306 return nil, errors.Wrap(err, "failed to create language config") 307 } 308 309 var contentChangeTracker *contentChangeMap 310 311 numWorkers := config.GetNumWorkerMultiplier() 312 if numWorkers > len(sites) { 313 numWorkers = len(sites) 314 } 315 var workers *para.Workers 316 if numWorkers > 1 { 317 workers = para.New(numWorkers) 318 } 319 320 h := &HugoSites{ 321 running: cfg.Running, 322 multilingual: langConfig, 323 multihost: cfg.Cfg.GetBool("multihost"), 324 Sites: sites, 325 workers: workers, 326 numWorkers: numWorkers, 327 skipRebuildForFilenames: make(map[string]bool), 328 init: &hugoSitesInit{ 329 data: lazy.New(), 330 layouts: lazy.New(), 331 gitInfo: lazy.New(), 332 translations: lazy.New(), 333 }, 334 } 335 336 h.fatalErrorHandler = &fatalErrorHandler{ 337 h: h, 338 donec: make(chan bool), 339 } 340 341 h.init.data.Add(func() (interface{}, error) { 342 err := h.loadData(h.PathSpec.BaseFs.Data.Dirs) 343 if err != nil { 344 return nil, errors.Wrap(err, "failed to load data") 345 } 346 return nil, nil 347 }) 348 349 h.init.layouts.Add(func() (interface{}, error) { 350 for _, s := range h.Sites { 351 if err := s.Tmpl().(tpl.TemplateManager).MarkReady(); err != nil { 352 return nil, err 353 } 354 } 355 return nil, nil 356 }) 357 358 h.init.translations.Add(func() (interface{}, error) { 359 if len(h.Sites) > 1 { 360 allTranslations := pagesToTranslationsMap(h.Sites) 361 assignTranslationsToPages(allTranslations, h.Sites) 362 } 363 364 return nil, nil 365 }) 366 367 h.init.gitInfo.Add(func() (interface{}, error) { 368 err := h.loadGitInfo() 369 if err != nil { 370 return nil, errors.Wrap(err, "failed to load Git info") 371 } 372 return nil, nil 373 }) 374 375 for _, s := range sites { 376 s.h = h 377 } 378 379 var l configLoader 380 if err := l.applyDeps(cfg, sites...); err != nil { 381 initErr = errors.Wrap(err, "add site dependencies") 382 } 383 384 h.Deps = sites[0].Deps 385 386 // Only needed in server mode. 387 // TODO(bep) clean up the running vs watching terms 388 if cfg.Running { 389 contentChangeTracker = &contentChangeMap{ 390 pathSpec: h.PathSpec, 391 symContent: make(map[string]map[string]bool), 392 leafBundles: radix.New(), 393 branchBundles: make(map[string]bool), 394 } 395 h.ContentChanges = contentChangeTracker 396 } 397 398 return h, initErr 399 } 400 401 func (h *HugoSites) loadGitInfo() error { 402 if h.Cfg.GetBool("enableGitInfo") { 403 gi, err := newGitInfo(h.Cfg) 404 if err != nil { 405 h.Log.Errorln("Failed to read Git log:", err) 406 } else { 407 h.gitInfo = gi 408 } 409 } 410 return nil 411 } 412 413 func (l configLoader) applyDeps(cfg deps.DepsCfg, sites ...*Site) error { 414 if cfg.TemplateProvider == nil { 415 cfg.TemplateProvider = tplimpl.DefaultTemplateProvider 416 } 417 418 if cfg.TranslationProvider == nil { 419 cfg.TranslationProvider = i18n.NewTranslationProvider() 420 } 421 422 var ( 423 d *deps.Deps 424 err error 425 ) 426 427 for _, s := range sites { 428 if s.Deps != nil { 429 continue 430 } 431 432 onCreated := func(d *deps.Deps) error { 433 s.Deps = d 434 435 // Set up the main publishing chain. 436 pub, err := publisher.NewDestinationPublisher( 437 d.ResourceSpec, 438 s.outputFormatsConfig, 439 s.mediaTypesConfig, 440 ) 441 if err != nil { 442 return err 443 } 444 s.publisher = pub 445 446 if err := s.initializeSiteInfo(); err != nil { 447 return err 448 } 449 450 d.Site = s.Info 451 452 siteConfig, err := l.loadSiteConfig(s.language) 453 if err != nil { 454 return errors.Wrap(err, "load site config") 455 } 456 s.siteConfigConfig = siteConfig 457 458 pm := &pageMap{ 459 contentMap: newContentMap(contentMapConfig{ 460 lang: s.Lang(), 461 taxonomyConfig: s.siteCfg.taxonomiesConfig.Values(), 462 taxonomyDisabled: !s.isEnabled(page.KindTerm), 463 taxonomyTermDisabled: !s.isEnabled(page.KindTaxonomy), 464 pageDisabled: !s.isEnabled(page.KindPage), 465 }), 466 s: s, 467 } 468 469 s.PageCollections = newPageCollections(pm) 470 471 s.siteRefLinker, err = newSiteRefLinker(s.language, s) 472 return err 473 } 474 475 cfg.Language = s.language 476 cfg.MediaTypes = s.mediaTypesConfig 477 cfg.OutputFormats = s.outputFormatsConfig 478 479 if d == nil { 480 cfg.WithTemplate = s.withSiteTemplates(cfg.WithTemplate) 481 482 var err error 483 d, err = deps.New(cfg) 484 if err != nil { 485 return errors.Wrap(err, "create deps") 486 } 487 488 d.OutputFormatsConfig = s.outputFormatsConfig 489 490 if err := onCreated(d); err != nil { 491 return errors.Wrap(err, "on created") 492 } 493 494 if err = d.LoadResources(); err != nil { 495 return errors.Wrap(err, "load resources") 496 } 497 498 } else { 499 d, err = d.ForLanguage(cfg, onCreated) 500 if err != nil { 501 return err 502 } 503 d.OutputFormatsConfig = s.outputFormatsConfig 504 } 505 } 506 507 return nil 508 } 509 510 // NewHugoSites creates HugoSites from the given config. 511 func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { 512 if cfg.Logger == nil { 513 cfg.Logger = loggers.NewErrorLogger() 514 } 515 sites, err := createSitesFromConfig(cfg) 516 if err != nil { 517 return nil, errors.Wrap(err, "from config") 518 } 519 return newHugoSites(cfg, sites...) 520 } 521 522 func (s *Site) withSiteTemplates(withTemplates ...func(templ tpl.TemplateManager) error) func(templ tpl.TemplateManager) error { 523 return func(templ tpl.TemplateManager) error { 524 for _, wt := range withTemplates { 525 if wt == nil { 526 continue 527 } 528 if err := wt(templ); err != nil { 529 return err 530 } 531 } 532 533 return nil 534 } 535 } 536 537 func createSitesFromConfig(cfg deps.DepsCfg) ([]*Site, error) { 538 var sites []*Site 539 540 languages := getLanguages(cfg.Cfg) 541 542 for _, lang := range languages { 543 if lang.Disabled { 544 continue 545 } 546 var s *Site 547 var err error 548 cfg.Language = lang 549 s, err = newSite(cfg) 550 551 if err != nil { 552 return nil, err 553 } 554 555 sites = append(sites, s) 556 } 557 558 return sites, nil 559 } 560 561 // Reset resets the sites and template caches etc., making it ready for a full rebuild. 562 func (h *HugoSites) reset(config *BuildCfg) { 563 if config.ResetState { 564 for i, s := range h.Sites { 565 h.Sites[i] = s.reset() 566 if r, ok := s.Fs.Destination.(hugofs.Reseter); ok { 567 r.Reset() 568 } 569 } 570 } 571 572 h.fatalErrorHandler = &fatalErrorHandler{ 573 h: h, 574 donec: make(chan bool), 575 } 576 577 h.init.Reset() 578 } 579 580 // resetLogs resets the log counters etc. Used to do a new build on the same sites. 581 func (h *HugoSites) resetLogs() { 582 h.Log.Reset() 583 loggers.GlobalErrorCounter.Reset() 584 for _, s := range h.Sites { 585 s.Deps.Log.Reset() 586 s.Deps.LogDistinct.Reset() 587 } 588 } 589 590 func (h *HugoSites) withSite(fn func(s *Site) error) error { 591 if h.workers == nil { 592 for _, s := range h.Sites { 593 if err := fn(s); err != nil { 594 return err 595 } 596 } 597 return nil 598 } 599 600 g, _ := h.workers.Start(context.Background()) 601 for _, s := range h.Sites { 602 s := s 603 g.Run(func() error { 604 return fn(s) 605 }) 606 } 607 return g.Wait() 608 } 609 610 func (h *HugoSites) createSitesFromConfig(cfg config.Provider) error { 611 oldLangs, _ := h.Cfg.Get("languagesSorted").(langs.Languages) 612 613 l := configLoader{cfg: h.Cfg} 614 if err := l.loadLanguageSettings(oldLangs); err != nil { 615 return err 616 } 617 618 depsCfg := deps.DepsCfg{Fs: h.Fs, Cfg: l.cfg} 619 620 sites, err := createSitesFromConfig(depsCfg) 621 if err != nil { 622 return err 623 } 624 625 langConfig, err := newMultiLingualFromSites(depsCfg.Cfg, sites...) 626 if err != nil { 627 return err 628 } 629 630 h.Sites = sites 631 632 for _, s := range sites { 633 s.h = h 634 } 635 636 var cl configLoader 637 if err := cl.applyDeps(depsCfg, sites...); err != nil { 638 return err 639 } 640 641 h.Deps = sites[0].Deps 642 643 h.multilingual = langConfig 644 h.multihost = h.Deps.Cfg.GetBool("multihost") 645 646 return nil 647 } 648 649 func (h *HugoSites) toSiteInfos() []*SiteInfo { 650 infos := make([]*SiteInfo, len(h.Sites)) 651 for i, s := range h.Sites { 652 infos[i] = s.Info 653 } 654 return infos 655 } 656 657 // BuildCfg holds build options used to, as an example, skip the render step. 658 type BuildCfg struct { 659 // Reset site state before build. Use to force full rebuilds. 660 ResetState bool 661 // If set, we re-create the sites from the given configuration before a build. 662 // This is needed if new languages are added. 663 NewConfig config.Provider 664 // Skip rendering. Useful for testing. 665 SkipRender bool 666 // Use this to indicate what changed (for rebuilds). 667 whatChanged *whatChanged 668 669 // This is a partial re-render of some selected pages. This means 670 // we should skip most of the processing. 671 PartialReRender bool 672 673 // Set in server mode when the last build failed for some reason. 674 ErrRecovery bool 675 676 // Recently visited URLs. This is used for partial re-rendering. 677 RecentlyVisited map[string]bool 678 679 // Can be set to build only with a sub set of the content source. 680 ContentInclusionFilter *glob.FilenameFilter 681 682 // Set when the buildlock is already acquired (e.g. the archetype content builder). 683 NoBuildLock bool 684 685 testCounters *testCounters 686 } 687 688 // shouldRender is used in the Fast Render Mode to determine if we need to re-render 689 // a Page: If it is recently visited (the home pages will always be in this set) or changed. 690 // Note that a page does not have to have a content page / file. 691 // For regular builds, this will allways return true. 692 // TODO(bep) rename/work this. 693 func (cfg *BuildCfg) shouldRender(p *pageState) bool { 694 if p.forceRender { 695 return true 696 } 697 698 if len(cfg.RecentlyVisited) == 0 { 699 return true 700 } 701 702 if cfg.RecentlyVisited[p.RelPermalink()] { 703 return true 704 } 705 706 if cfg.whatChanged != nil && !p.File().IsZero() { 707 return cfg.whatChanged.files[p.File().Filename()] 708 } 709 710 return false 711 } 712 713 func (h *HugoSites) renderCrossSitesSitemap() error { 714 if !h.multilingual.enabled() || h.IsMultihost() { 715 return nil 716 } 717 718 sitemapEnabled := false 719 for _, s := range h.Sites { 720 if s.isEnabled(kindSitemap) { 721 sitemapEnabled = true 722 break 723 } 724 } 725 726 if !sitemapEnabled { 727 return nil 728 } 729 730 s := h.Sites[0] 731 732 templ := s.lookupLayouts("sitemapindex.xml", "_default/sitemapindex.xml", "_internal/_default/sitemapindex.xml") 733 734 return s.renderAndWriteXML(&s.PathSpec.ProcessingStats.Sitemaps, "sitemapindex", 735 s.siteCfg.sitemap.Filename, h.toSiteInfos(), templ) 736 } 737 738 func (h *HugoSites) renderCrossSitesRobotsTXT() error { 739 if h.multihost { 740 return nil 741 } 742 if !h.Cfg.GetBool("enableRobotsTXT") { 743 return nil 744 } 745 746 s := h.Sites[0] 747 748 p, err := newPageStandalone(&pageMeta{ 749 s: s, 750 kind: kindRobotsTXT, 751 urlPaths: pagemeta.URLPath{ 752 URL: "robots.txt", 753 }, 754 }, 755 output.RobotsTxtFormat) 756 if err != nil { 757 return err 758 } 759 760 if !p.render { 761 return nil 762 } 763 764 templ := s.lookupLayouts("robots.txt", "_default/robots.txt", "_internal/_default/robots.txt") 765 766 return s.renderAndWritePage(&s.PathSpec.ProcessingStats.Pages, "Robots Txt", "robots.txt", p, templ) 767 } 768 769 func (h *HugoSites) removePageByFilename(filename string) { 770 h.getContentMaps().withMaps(func(m *pageMap) error { 771 m.deleteBundleMatching(func(b *contentNode) bool { 772 if b.p == nil { 773 return false 774 } 775 776 if b.fi == nil { 777 return false 778 } 779 780 return b.fi.Meta().Filename == filename 781 }) 782 return nil 783 }) 784 } 785 786 func (h *HugoSites) createPageCollections() error { 787 allPages := newLazyPagesFactory(func() page.Pages { 788 var pages page.Pages 789 for _, s := range h.Sites { 790 pages = append(pages, s.Pages()...) 791 } 792 793 page.SortByDefault(pages) 794 795 return pages 796 }) 797 798 allRegularPages := newLazyPagesFactory(func() page.Pages { 799 return h.findPagesByKindIn(page.KindPage, allPages.get()) 800 }) 801 802 for _, s := range h.Sites { 803 s.PageCollections.allPages = allPages 804 s.PageCollections.allRegularPages = allRegularPages 805 } 806 807 return nil 808 } 809 810 func (s *Site) preparePagesForRender(isRenderingSite bool, idx int) error { 811 var err error 812 s.pageMap.withEveryBundlePage(func(p *pageState) bool { 813 if err = p.initOutputFormat(isRenderingSite, idx); err != nil { 814 return true 815 } 816 return false 817 }) 818 return nil 819 } 820 821 // Pages returns all pages for all sites. 822 func (h *HugoSites) Pages() page.Pages { 823 return h.Sites[0].AllPages() 824 } 825 826 func (h *HugoSites) loadData(fis []hugofs.FileMetaInfo) (err error) { 827 spec := source.NewSourceSpec(h.PathSpec, nil, nil) 828 829 h.data = make(map[string]interface{}) 830 for _, fi := range fis { 831 fileSystem := spec.NewFilesystemFromFileMetaInfo(fi) 832 files, err := fileSystem.Files() 833 if err != nil { 834 return err 835 } 836 for _, r := range files { 837 if err := h.handleDataFile(r); err != nil { 838 return err 839 } 840 } 841 } 842 843 return 844 } 845 846 func (h *HugoSites) handleDataFile(r source.File) error { 847 var current map[string]interface{} 848 849 f, err := r.FileInfo().Meta().Open() 850 if err != nil { 851 return errors.Wrapf(err, "data: failed to open %q:", r.LogicalName()) 852 } 853 defer f.Close() 854 855 // Crawl in data tree to insert data 856 current = h.data 857 keyParts := strings.Split(r.Dir(), helpers.FilePathSeparator) 858 859 for _, key := range keyParts { 860 if key != "" { 861 if _, ok := current[key]; !ok { 862 current[key] = make(map[string]interface{}) 863 } 864 current = current[key].(map[string]interface{}) 865 } 866 } 867 868 data, err := h.readData(r) 869 if err != nil { 870 return h.errWithFileContext(err, r) 871 } 872 873 if data == nil { 874 return nil 875 } 876 877 // filepath.Walk walks the files in lexical order, '/' comes before '.' 878 higherPrecedentData := current[r.BaseFileName()] 879 880 switch data.(type) { 881 case nil: 882 case map[string]interface{}: 883 884 switch higherPrecedentData.(type) { 885 case nil: 886 current[r.BaseFileName()] = data 887 case map[string]interface{}: 888 // merge maps: insert entries from data for keys that 889 // don't already exist in higherPrecedentData 890 higherPrecedentMap := higherPrecedentData.(map[string]interface{}) 891 for key, value := range data.(map[string]interface{}) { 892 if _, exists := higherPrecedentMap[key]; exists { 893 // this warning could happen if 894 // 1. A theme uses the same key; the main data folder wins 895 // 2. A sub folder uses the same key: the sub folder wins 896 // TODO(bep) figure out a way to detect 2) above and make that a WARN 897 h.Log.Infof("Data for key '%s' in path '%s' is overridden by higher precedence data already in the data tree", key, r.Path()) 898 } else { 899 higherPrecedentMap[key] = value 900 } 901 } 902 default: 903 // can't merge: higherPrecedentData is not a map 904 h.Log.Warnf("The %T data from '%s' overridden by "+ 905 "higher precedence %T data already in the data tree", data, r.Path(), higherPrecedentData) 906 } 907 908 case []interface{}: 909 if higherPrecedentData == nil { 910 current[r.BaseFileName()] = data 911 } else { 912 // we don't merge array data 913 h.Log.Warnf("The %T data from '%s' overridden by "+ 914 "higher precedence %T data already in the data tree", data, r.Path(), higherPrecedentData) 915 } 916 917 default: 918 h.Log.Errorf("unexpected data type %T in file %s", data, r.LogicalName()) 919 } 920 921 return nil 922 } 923 924 func (h *HugoSites) errWithFileContext(err error, f source.File) error { 925 fim, ok := f.FileInfo().(hugofs.FileMetaInfo) 926 if !ok { 927 return err 928 } 929 930 realFilename := fim.Meta().Filename 931 932 err, _ = herrors.WithFileContextForFile( 933 err, 934 realFilename, 935 realFilename, 936 h.SourceSpec.Fs.Source, 937 herrors.SimpleLineMatcher) 938 939 return err 940 } 941 942 func (h *HugoSites) readData(f source.File) (interface{}, error) { 943 file, err := f.FileInfo().Meta().Open() 944 if err != nil { 945 return nil, errors.Wrap(err, "readData: failed to open data file") 946 } 947 defer file.Close() 948 content := helpers.ReaderToBytes(file) 949 950 format := metadecoders.FormatFromString(f.Extension()) 951 return metadecoders.Default.Unmarshal(content, format) 952 } 953 954 func (h *HugoSites) findPagesByKindIn(kind string, inPages page.Pages) page.Pages { 955 return h.Sites[0].findPagesByKindIn(kind, inPages) 956 } 957 958 func (h *HugoSites) resetPageState() { 959 h.getContentMaps().walkBundles(func(n *contentNode) bool { 960 if n.p == nil { 961 return false 962 } 963 p := n.p 964 for _, po := range p.pageOutputs { 965 if po.cp == nil { 966 continue 967 } 968 po.cp.Reset() 969 } 970 971 return false 972 }) 973 } 974 975 func (h *HugoSites) resetPageStateFromEvents(idset identity.Identities) { 976 h.getContentMaps().walkBundles(func(n *contentNode) bool { 977 if n.p == nil { 978 return false 979 } 980 p := n.p 981 OUTPUTS: 982 for _, po := range p.pageOutputs { 983 if po.cp == nil { 984 continue 985 } 986 for id := range idset { 987 if po.cp.dependencyTracker.Search(id) != nil { 988 po.cp.Reset() 989 continue OUTPUTS 990 } 991 } 992 } 993 994 if p.shortcodeState == nil { 995 return false 996 } 997 998 for _, s := range p.shortcodeState.shortcodes { 999 for _, templ := range s.templs { 1000 sid := templ.(identity.Manager) 1001 for id := range idset { 1002 if sid.Search(id) != nil { 1003 for _, po := range p.pageOutputs { 1004 if po.cp != nil { 1005 po.cp.Reset() 1006 } 1007 } 1008 return false 1009 } 1010 } 1011 } 1012 } 1013 return false 1014 }) 1015 } 1016 1017 // Used in partial reloading to determine if the change is in a bundle. 1018 type contentChangeMap struct { 1019 mu sync.RWMutex 1020 1021 // Holds directories with leaf bundles. 1022 leafBundles *radix.Tree 1023 1024 // Holds directories with branch bundles. 1025 branchBundles map[string]bool 1026 1027 pathSpec *helpers.PathSpec 1028 1029 // Hugo supports symlinked content (both directories and files). This 1030 // can lead to situations where the same file can be referenced from several 1031 // locations in /content -- which is really cool, but also means we have to 1032 // go an extra mile to handle changes. 1033 // This map is only used in watch mode. 1034 // It maps either file to files or the real dir to a set of content directories 1035 // where it is in use. 1036 symContentMu sync.Mutex 1037 symContent map[string]map[string]bool 1038 } 1039 1040 func (m *contentChangeMap) add(dirname string, tp bundleDirType) { 1041 m.mu.Lock() 1042 if !strings.HasSuffix(dirname, helpers.FilePathSeparator) { 1043 dirname += helpers.FilePathSeparator 1044 } 1045 switch tp { 1046 case bundleBranch: 1047 m.branchBundles[dirname] = true 1048 case bundleLeaf: 1049 m.leafBundles.Insert(dirname, true) 1050 default: 1051 m.mu.Unlock() 1052 panic("invalid bundle type") 1053 } 1054 m.mu.Unlock() 1055 } 1056 1057 func (m *contentChangeMap) resolveAndRemove(filename string) (string, bundleDirType) { 1058 m.mu.RLock() 1059 defer m.mu.RUnlock() 1060 1061 // Bundles share resources, so we need to start from the virtual root. 1062 relFilename := m.pathSpec.RelContentDir(filename) 1063 dir, name := filepath.Split(relFilename) 1064 if !strings.HasSuffix(dir, helpers.FilePathSeparator) { 1065 dir += helpers.FilePathSeparator 1066 } 1067 1068 if _, found := m.branchBundles[dir]; found { 1069 delete(m.branchBundles, dir) 1070 return dir, bundleBranch 1071 } 1072 1073 if key, _, found := m.leafBundles.LongestPrefix(dir); found { 1074 m.leafBundles.Delete(key) 1075 dir = string(key) 1076 return dir, bundleLeaf 1077 } 1078 1079 fileTp, isContent := classifyBundledFile(name) 1080 if isContent && fileTp != bundleNot { 1081 // A new bundle. 1082 return dir, fileTp 1083 } 1084 1085 return dir, bundleNot 1086 } 1087 1088 func (m *contentChangeMap) addSymbolicLinkMapping(fim hugofs.FileMetaInfo) { 1089 meta := fim.Meta() 1090 if !meta.IsSymlink { 1091 return 1092 } 1093 m.symContentMu.Lock() 1094 1095 from, to := meta.Filename, meta.OriginalFilename 1096 if fim.IsDir() { 1097 if !strings.HasSuffix(from, helpers.FilePathSeparator) { 1098 from += helpers.FilePathSeparator 1099 } 1100 } 1101 1102 mm, found := m.symContent[from] 1103 1104 if !found { 1105 mm = make(map[string]bool) 1106 m.symContent[from] = mm 1107 } 1108 mm[to] = true 1109 m.symContentMu.Unlock() 1110 } 1111 1112 func (m *contentChangeMap) GetSymbolicLinkMappings(dir string) []string { 1113 mm, found := m.symContent[dir] 1114 if !found { 1115 return nil 1116 } 1117 dirs := make([]string, len(mm)) 1118 i := 0 1119 for dir := range mm { 1120 dirs[i] = dir 1121 i++ 1122 } 1123 1124 sort.Strings(dirs) 1125 1126 return dirs 1127 }