github.com/kristoff-it/hugo@v0.47.1/hugolib/hugo_sites.go (about) 1 // Copyright 2016-present The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package hugolib 15 16 import ( 17 "errors" 18 "io" 19 "path/filepath" 20 "sort" 21 "strings" 22 "sync" 23 24 "github.com/gohugoio/hugo/deps" 25 "github.com/gohugoio/hugo/helpers" 26 "github.com/gohugoio/hugo/langs" 27 "github.com/gohugoio/hugo/publisher" 28 29 "github.com/gohugoio/hugo/i18n" 30 "github.com/gohugoio/hugo/tpl" 31 "github.com/gohugoio/hugo/tpl/tplimpl" 32 jww "github.com/spf13/jwalterweatherman" 33 ) 34 35 // HugoSites represents the sites to build. Each site represents a language. 36 type HugoSites struct { 37 Sites []*Site 38 39 multilingual *Multilingual 40 41 // Multihost is set if multilingual and baseURL set on the language level. 42 multihost bool 43 44 // If this is running in the dev server. 45 running bool 46 47 *deps.Deps 48 49 // Keeps track of bundle directories and symlinks to enable partial rebuilding. 50 ContentChanges *contentChangeMap 51 52 // If enabled, keeps a revision map for all content. 53 gitInfo *gitInfo 54 } 55 56 func (h *HugoSites) IsMultihost() bool { 57 return h != nil && h.multihost 58 } 59 60 func (h *HugoSites) NumLogErrors() int { 61 if h == nil { 62 return 0 63 } 64 return int(h.Log.LogCountForLevelsGreaterThanorEqualTo(jww.LevelError)) 65 } 66 67 func (h *HugoSites) PrintProcessingStats(w io.Writer) { 68 stats := make([]*helpers.ProcessingStats, len(h.Sites)) 69 for i := 0; i < len(h.Sites); i++ { 70 stats[i] = h.Sites[i].PathSpec.ProcessingStats 71 } 72 helpers.ProcessingStatsTable(w, stats...) 73 } 74 75 func (h *HugoSites) langSite() map[string]*Site { 76 m := make(map[string]*Site) 77 for _, s := range h.Sites { 78 m[s.Language.Lang] = s 79 } 80 return m 81 } 82 83 // GetContentPage finds a Page with content given the absolute filename. 84 // Returns nil if none found. 85 func (h *HugoSites) GetContentPage(filename string) *Page { 86 for _, s := range h.Sites { 87 pos := s.rawAllPages.findPagePosByFilename(filename) 88 if pos == -1 { 89 continue 90 } 91 return s.rawAllPages[pos] 92 } 93 94 // If not found already, this may be bundled in another content file. 95 dir := filepath.Dir(filename) 96 97 for _, s := range h.Sites { 98 pos := s.rawAllPages.findPagePosByFilnamePrefix(dir) 99 if pos == -1 { 100 continue 101 } 102 return s.rawAllPages[pos] 103 } 104 return nil 105 } 106 107 // NewHugoSites creates a new collection of sites given the input sites, building 108 // a language configuration based on those. 109 func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { 110 111 if cfg.Language != nil { 112 return nil, errors.New("Cannot provide Language in Cfg when sites are provided") 113 } 114 115 langConfig, err := newMultiLingualFromSites(cfg.Cfg, sites...) 116 117 if err != nil { 118 return nil, err 119 } 120 121 var contentChangeTracker *contentChangeMap 122 123 h := &HugoSites{ 124 running: cfg.Running, 125 multilingual: langConfig, 126 multihost: cfg.Cfg.GetBool("multihost"), 127 Sites: sites} 128 129 for _, s := range sites { 130 s.owner = h 131 } 132 133 if err := applyDeps(cfg, sites...); err != nil { 134 return nil, err 135 } 136 137 h.Deps = sites[0].Deps 138 139 // Only needed in server mode. 140 // TODO(bep) clean up the running vs watching terms 141 if cfg.Running { 142 contentChangeTracker = &contentChangeMap{pathSpec: h.PathSpec, symContent: make(map[string]map[string]bool)} 143 h.ContentChanges = contentChangeTracker 144 } 145 146 if err := h.initGitInfo(); err != nil { 147 return nil, err 148 } 149 150 return h, nil 151 } 152 153 func (h *HugoSites) initGitInfo() error { 154 if h.Cfg.GetBool("enableGitInfo") { 155 gi, err := newGitInfo(h.Cfg) 156 if err != nil { 157 h.Log.ERROR.Println("Failed to read Git log:", err) 158 } else { 159 h.gitInfo = gi 160 } 161 } 162 return nil 163 } 164 165 func applyDeps(cfg deps.DepsCfg, sites ...*Site) error { 166 if cfg.TemplateProvider == nil { 167 cfg.TemplateProvider = tplimpl.DefaultTemplateProvider 168 } 169 170 if cfg.TranslationProvider == nil { 171 cfg.TranslationProvider = i18n.NewTranslationProvider() 172 } 173 174 var ( 175 d *deps.Deps 176 err error 177 ) 178 179 for _, s := range sites { 180 if s.Deps != nil { 181 continue 182 } 183 184 cfg.Language = s.Language 185 cfg.MediaTypes = s.mediaTypesConfig 186 cfg.OutputFormats = s.outputFormatsConfig 187 188 if d == nil { 189 cfg.WithTemplate = s.withSiteTemplates(cfg.WithTemplate) 190 191 var err error 192 d, err = deps.New(cfg) 193 if err != nil { 194 return err 195 } 196 197 d.OutputFormatsConfig = s.outputFormatsConfig 198 s.Deps = d 199 200 if err = d.LoadResources(); err != nil { 201 return err 202 } 203 204 } else { 205 d, err = d.ForLanguage(cfg) 206 if err != nil { 207 return err 208 } 209 d.OutputFormatsConfig = s.outputFormatsConfig 210 s.Deps = d 211 } 212 213 // Set up the main publishing chain. 214 s.publisher = publisher.NewDestinationPublisher(d.PathSpec.BaseFs.PublishFs, s.outputFormatsConfig, s.mediaTypesConfig, cfg.Cfg.GetBool("minify")) 215 216 if err := s.initializeSiteInfo(); err != nil { 217 return err 218 } 219 220 siteConfig, err := loadSiteConfig(s.Language) 221 if err != nil { 222 return err 223 } 224 s.siteConfig = siteConfig 225 s.siteRefLinker, err = newSiteRefLinker(s.Language, s) 226 if err != nil { 227 return err 228 } 229 } 230 231 return nil 232 } 233 234 // NewHugoSites creates HugoSites from the given config. 235 func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { 236 sites, err := createSitesFromConfig(cfg) 237 if err != nil { 238 return nil, err 239 } 240 return newHugoSites(cfg, sites...) 241 } 242 243 func (s *Site) withSiteTemplates(withTemplates ...func(templ tpl.TemplateHandler) error) func(templ tpl.TemplateHandler) error { 244 return func(templ tpl.TemplateHandler) error { 245 templ.LoadTemplates("") 246 247 for _, wt := range withTemplates { 248 if wt == nil { 249 continue 250 } 251 if err := wt(templ); err != nil { 252 return err 253 } 254 } 255 256 return nil 257 } 258 } 259 260 func createSitesFromConfig(cfg deps.DepsCfg) ([]*Site, error) { 261 262 var ( 263 sites []*Site 264 ) 265 266 languages := getLanguages(cfg.Cfg) 267 268 for _, lang := range languages { 269 if lang.Disabled { 270 continue 271 } 272 var s *Site 273 var err error 274 cfg.Language = lang 275 s, err = newSite(cfg) 276 277 if err != nil { 278 return nil, err 279 } 280 281 sites = append(sites, s) 282 } 283 284 return sites, nil 285 } 286 287 // Reset resets the sites and template caches, making it ready for a full rebuild. 288 func (h *HugoSites) reset() { 289 for i, s := range h.Sites { 290 h.Sites[i] = s.reset() 291 } 292 } 293 294 // resetLogs resets the log counters etc. Used to do a new build on the same sites. 295 func (h *HugoSites) resetLogs() { 296 h.Log.ResetLogCounters() 297 for _, s := range h.Sites { 298 s.Deps.DistinctErrorLog = helpers.NewDistinctLogger(h.Log.ERROR) 299 } 300 } 301 302 func (h *HugoSites) createSitesFromConfig() error { 303 oldLangs, _ := h.Cfg.Get("languagesSorted").(langs.Languages) 304 305 if err := loadLanguageSettings(h.Cfg, oldLangs); err != nil { 306 return err 307 } 308 309 depsCfg := deps.DepsCfg{Fs: h.Fs, Cfg: h.Cfg} 310 311 sites, err := createSitesFromConfig(depsCfg) 312 313 if err != nil { 314 return err 315 } 316 317 langConfig, err := newMultiLingualFromSites(depsCfg.Cfg, sites...) 318 319 if err != nil { 320 return err 321 } 322 323 h.Sites = sites 324 325 for _, s := range sites { 326 s.owner = h 327 } 328 329 if err := applyDeps(depsCfg, sites...); err != nil { 330 return err 331 } 332 333 h.Deps = sites[0].Deps 334 335 h.multilingual = langConfig 336 h.multihost = h.Deps.Cfg.GetBool("multihost") 337 338 return nil 339 } 340 341 func (h *HugoSites) toSiteInfos() []*SiteInfo { 342 infos := make([]*SiteInfo, len(h.Sites)) 343 for i, s := range h.Sites { 344 infos[i] = &s.Info 345 } 346 return infos 347 } 348 349 // BuildCfg holds build options used to, as an example, skip the render step. 350 type BuildCfg struct { 351 // Reset site state before build. Use to force full rebuilds. 352 ResetState bool 353 // Re-creates the sites from configuration before a build. 354 // This is needed if new languages are added. 355 CreateSitesFromConfig bool 356 // Skip rendering. Useful for testing. 357 SkipRender bool 358 // Use this to indicate what changed (for rebuilds). 359 whatChanged *whatChanged 360 // Recently visited URLs. This is used for partial re-rendering. 361 RecentlyVisited map[string]bool 362 } 363 364 // shouldRender is used in the Fast Render Mode to determine if we need to re-render 365 // a Page: If it is recently visited (the home pages will always be in this set) or changed. 366 // Note that a page does not have to have a content page / file. 367 // For regular builds, this will allways return true. 368 func (cfg *BuildCfg) shouldRender(p *Page) bool { 369 if p.forceRender { 370 p.forceRender = false 371 return true 372 } 373 374 if len(cfg.RecentlyVisited) == 0 { 375 return true 376 } 377 378 if cfg.RecentlyVisited[p.RelPermalink()] { 379 return true 380 } 381 382 if cfg.whatChanged != nil && p.File != nil { 383 return cfg.whatChanged.files[p.File.Filename()] 384 } 385 386 return false 387 } 388 389 func (h *HugoSites) renderCrossSitesArtifacts() error { 390 391 if !h.multilingual.enabled() || h.IsMultihost() { 392 return nil 393 } 394 395 sitemapEnabled := false 396 for _, s := range h.Sites { 397 if s.isEnabled(kindSitemap) { 398 sitemapEnabled = true 399 break 400 } 401 } 402 403 if !sitemapEnabled { 404 return nil 405 } 406 407 // TODO(bep) DRY 408 sitemapDefault := parseSitemap(h.Cfg.GetStringMap("sitemap")) 409 410 s := h.Sites[0] 411 412 smLayouts := []string{"sitemapindex.xml", "_default/sitemapindex.xml", "_internal/_default/sitemapindex.xml"} 413 414 return s.renderAndWriteXML(&s.PathSpec.ProcessingStats.Sitemaps, "sitemapindex", 415 sitemapDefault.Filename, h.toSiteInfos(), s.appendThemeTemplates(smLayouts)...) 416 } 417 418 func (h *HugoSites) assignMissingTranslations() error { 419 420 // This looks heavy, but it should be a small number of nodes by now. 421 allPages := h.findAllPagesByKindNotIn(KindPage) 422 for _, nodeType := range []string{KindHome, KindSection, KindTaxonomy, KindTaxonomyTerm} { 423 nodes := h.findPagesByKindIn(nodeType, allPages) 424 425 // Assign translations 426 for _, t1 := range nodes { 427 for _, t2 := range nodes { 428 if t1.isNewTranslation(t2) { 429 t1.translations = append(t1.translations, t2) 430 } 431 } 432 } 433 } 434 435 // Now we can sort the translations. 436 for _, p := range allPages { 437 if len(p.translations) > 0 { 438 pageBy(languagePageSort).Sort(p.translations) 439 } 440 } 441 return nil 442 443 } 444 445 // createMissingPages creates home page, taxonomies etc. that isnt't created as an 446 // effect of having a content file. 447 func (h *HugoSites) createMissingPages() error { 448 var newPages Pages 449 450 for _, s := range h.Sites { 451 if s.isEnabled(KindHome) { 452 // home pages 453 home := s.findPagesByKind(KindHome) 454 if len(home) > 1 { 455 panic("Too many homes") 456 } 457 if len(home) == 0 { 458 n := s.newHomePage() 459 s.Pages = append(s.Pages, n) 460 newPages = append(newPages, n) 461 } 462 } 463 464 // Will create content-less root sections. 465 newSections := s.assembleSections() 466 s.Pages = append(s.Pages, newSections...) 467 newPages = append(newPages, newSections...) 468 469 // taxonomy list and terms pages 470 taxonomies := s.Language.GetStringMapString("taxonomies") 471 if len(taxonomies) > 0 { 472 taxonomyPages := s.findPagesByKind(KindTaxonomy) 473 taxonomyTermsPages := s.findPagesByKind(KindTaxonomyTerm) 474 for _, plural := range taxonomies { 475 if s.isEnabled(KindTaxonomyTerm) { 476 foundTaxonomyTermsPage := false 477 for _, p := range taxonomyTermsPages { 478 if p.sections[0] == plural { 479 foundTaxonomyTermsPage = true 480 break 481 } 482 } 483 484 if !foundTaxonomyTermsPage { 485 n := s.newTaxonomyTermsPage(plural) 486 s.Pages = append(s.Pages, n) 487 newPages = append(newPages, n) 488 } 489 } 490 491 if s.isEnabled(KindTaxonomy) { 492 for key := range s.Taxonomies[plural] { 493 foundTaxonomyPage := false 494 origKey := key 495 496 if s.Info.preserveTaxonomyNames { 497 key = s.PathSpec.MakePathSanitized(key) 498 } 499 for _, p := range taxonomyPages { 500 // Some people may have /authors/MaxMustermann etc. as paths. 501 // p.sections contains the raw values from the file system. 502 // See https://github.com/gohugoio/hugo/issues/4238 503 singularKey := s.PathSpec.MakePathSanitized(p.sections[1]) 504 if p.sections[0] == plural && singularKey == key { 505 foundTaxonomyPage = true 506 break 507 } 508 } 509 510 if !foundTaxonomyPage { 511 n := s.newTaxonomyPage(plural, origKey) 512 s.Pages = append(s.Pages, n) 513 newPages = append(newPages, n) 514 } 515 } 516 } 517 } 518 } 519 } 520 521 if len(newPages) > 0 { 522 // This resorting is unfortunate, but it also needs to be sorted 523 // when sections are created. 524 first := h.Sites[0] 525 526 first.AllPages = append(first.AllPages, newPages...) 527 528 first.AllPages.Sort() 529 530 for _, s := range h.Sites { 531 s.Pages.Sort() 532 } 533 534 for i := 1; i < len(h.Sites); i++ { 535 h.Sites[i].AllPages = first.AllPages 536 } 537 } 538 539 return nil 540 } 541 542 func (h *HugoSites) removePageByFilename(filename string) { 543 for _, s := range h.Sites { 544 s.removePageFilename(filename) 545 } 546 } 547 548 func (h *HugoSites) setupTranslations() { 549 for _, s := range h.Sites { 550 for _, p := range s.rawAllPages { 551 if p.Kind == kindUnknown { 552 p.Kind = p.s.kindFromSections(p.sections) 553 } 554 555 if !p.s.isEnabled(p.Kind) { 556 continue 557 } 558 559 shouldBuild := p.shouldBuild() 560 s.updateBuildStats(p) 561 if shouldBuild { 562 if p.headless { 563 s.headlessPages = append(s.headlessPages, p) 564 } else { 565 s.Pages = append(s.Pages, p) 566 } 567 } 568 } 569 } 570 571 allPages := make(Pages, 0) 572 573 for _, s := range h.Sites { 574 allPages = append(allPages, s.Pages...) 575 } 576 577 allPages.Sort() 578 579 for _, s := range h.Sites { 580 s.AllPages = allPages 581 } 582 583 // Pull over the collections from the master site 584 for i := 1; i < len(h.Sites); i++ { 585 h.Sites[i].Data = h.Sites[0].Data 586 } 587 588 if len(h.Sites) > 1 { 589 allTranslations := pagesToTranslationsMap(allPages) 590 assignTranslationsToPages(allTranslations, allPages) 591 } 592 } 593 594 func (s *Site) preparePagesForRender(start bool) error { 595 for _, p := range s.Pages { 596 p.setContentInit(start) 597 if err := p.initMainOutputFormat(); err != nil { 598 return err 599 } 600 } 601 602 for _, p := range s.headlessPages { 603 p.setContentInit(start) 604 if err := p.initMainOutputFormat(); err != nil { 605 return err 606 } 607 } 608 609 return nil 610 } 611 612 // Pages returns all pages for all sites. 613 func (h *HugoSites) Pages() Pages { 614 return h.Sites[0].AllPages 615 } 616 617 func handleShortcodes(p *PageWithoutContent, rawContentCopy []byte) ([]byte, error) { 618 if p.shortcodeState != nil && p.shortcodeState.contentShortcodes.Len() > 0 { 619 p.s.Log.DEBUG.Printf("Replace %d shortcodes in %q", p.shortcodeState.contentShortcodes.Len(), p.BaseFileName()) 620 err := p.shortcodeState.executeShortcodesForDelta(p) 621 622 if err != nil { 623 return rawContentCopy, err 624 } 625 626 rawContentCopy, err = replaceShortcodeTokens(rawContentCopy, shortcodePlaceholderPrefix, p.shortcodeState.renderedShortcodes) 627 628 if err != nil { 629 p.s.Log.FATAL.Printf("Failed to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error()) 630 } 631 } 632 633 return rawContentCopy, nil 634 } 635 636 func (s *Site) updateBuildStats(page *Page) { 637 if page.IsDraft() { 638 s.draftCount++ 639 } 640 641 if page.IsFuture() { 642 s.futureCount++ 643 } 644 645 if page.IsExpired() { 646 s.expiredCount++ 647 } 648 } 649 650 func (h *HugoSites) findPagesByKindNotIn(kind string, inPages Pages) Pages { 651 return h.Sites[0].findPagesByKindNotIn(kind, inPages) 652 } 653 654 func (h *HugoSites) findPagesByKindIn(kind string, inPages Pages) Pages { 655 return h.Sites[0].findPagesByKindIn(kind, inPages) 656 } 657 658 func (h *HugoSites) findAllPagesByKind(kind string) Pages { 659 return h.findPagesByKindIn(kind, h.Sites[0].AllPages) 660 } 661 662 func (h *HugoSites) findAllPagesByKindNotIn(kind string) Pages { 663 return h.findPagesByKindNotIn(kind, h.Sites[0].AllPages) 664 } 665 666 func (h *HugoSites) findPagesByShortcode(shortcode string) Pages { 667 var pages Pages 668 for _, s := range h.Sites { 669 pages = append(pages, s.findPagesByShortcode(shortcode)...) 670 } 671 return pages 672 } 673 674 // Used in partial reloading to determine if the change is in a bundle. 675 type contentChangeMap struct { 676 mu sync.RWMutex 677 branches []string 678 leafs []string 679 680 pathSpec *helpers.PathSpec 681 682 // Hugo supports symlinked content (both directories and files). This 683 // can lead to situations where the same file can be referenced from several 684 // locations in /content -- which is really cool, but also means we have to 685 // go an extra mile to handle changes. 686 // This map is only used in watch mode. 687 // It maps either file to files or the real dir to a set of content directories where it is in use. 688 symContent map[string]map[string]bool 689 symContentMu sync.Mutex 690 } 691 692 func (m *contentChangeMap) add(filename string, tp bundleDirType) { 693 m.mu.Lock() 694 dir := filepath.Dir(filename) + helpers.FilePathSeparator 695 dir = strings.TrimPrefix(dir, ".") 696 switch tp { 697 case bundleBranch: 698 m.branches = append(m.branches, dir) 699 case bundleLeaf: 700 m.leafs = append(m.leafs, dir) 701 default: 702 panic("invalid bundle type") 703 } 704 m.mu.Unlock() 705 } 706 707 // Track the addition of bundle dirs. 708 func (m *contentChangeMap) handleBundles(b *bundleDirs) { 709 for _, bd := range b.bundles { 710 m.add(bd.fi.Path(), bd.tp) 711 } 712 } 713 714 // resolveAndRemove resolves the given filename to the root folder of a bundle, if relevant. 715 // It also removes the entry from the map. It will be re-added again by the partial 716 // build if it still is a bundle. 717 func (m *contentChangeMap) resolveAndRemove(filename string) (string, string, bundleDirType) { 718 m.mu.RLock() 719 defer m.mu.RUnlock() 720 721 // Bundles share resources, so we need to start from the virtual root. 722 relPath := m.pathSpec.RelContentDir(filename) 723 dir, name := filepath.Split(relPath) 724 if !strings.HasSuffix(dir, helpers.FilePathSeparator) { 725 dir += helpers.FilePathSeparator 726 } 727 728 fileTp, isContent := classifyBundledFile(name) 729 730 // This may be a member of a bundle. Start with branch bundles, the most specific. 731 if fileTp == bundleBranch || (fileTp == bundleNot && !isContent) { 732 for i, b := range m.branches { 733 if b == dir { 734 m.branches = append(m.branches[:i], m.branches[i+1:]...) 735 return dir, b, bundleBranch 736 } 737 } 738 } 739 740 // And finally the leaf bundles, which can contain anything. 741 for i, l := range m.leafs { 742 if strings.HasPrefix(dir, l) { 743 m.leafs = append(m.leafs[:i], m.leafs[i+1:]...) 744 return dir, l, bundleLeaf 745 } 746 } 747 748 if isContent && fileTp != bundleNot { 749 // A new bundle. 750 return dir, dir, fileTp 751 } 752 753 // Not part of any bundle 754 return dir, filename, bundleNot 755 } 756 757 func (m *contentChangeMap) addSymbolicLinkMapping(from, to string) { 758 m.symContentMu.Lock() 759 mm, found := m.symContent[from] 760 if !found { 761 mm = make(map[string]bool) 762 m.symContent[from] = mm 763 } 764 mm[to] = true 765 m.symContentMu.Unlock() 766 } 767 768 func (m *contentChangeMap) GetSymbolicLinkMappings(dir string) []string { 769 mm, found := m.symContent[dir] 770 if !found { 771 return nil 772 } 773 dirs := make([]string, len(mm)) 774 i := 0 775 for dir := range mm { 776 dirs[i] = dir 777 i++ 778 } 779 780 sort.Strings(dirs) 781 return dirs 782 }