github.com/olliephillips/hugo@v0.42.2/hugolib/hugo_sites.go (about) 1 // Copyright 2016-present The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package hugolib 15 16 import ( 17 "errors" 18 "io" 19 "path/filepath" 20 "sort" 21 "strings" 22 "sync" 23 24 "github.com/gohugoio/hugo/resource" 25 26 "github.com/gohugoio/hugo/deps" 27 "github.com/gohugoio/hugo/helpers" 28 "github.com/gohugoio/hugo/langs" 29 30 "github.com/gohugoio/hugo/i18n" 31 "github.com/gohugoio/hugo/tpl" 32 "github.com/gohugoio/hugo/tpl/tplimpl" 33 jww "github.com/spf13/jwalterweatherman" 34 ) 35 36 // HugoSites represents the sites to build. Each site represents a language. 37 type HugoSites struct { 38 Sites []*Site 39 40 multilingual *Multilingual 41 42 // Multihost is set if multilingual and baseURL set on the language level. 43 multihost bool 44 45 // If this is running in the dev server. 46 running bool 47 48 *deps.Deps 49 50 // Keeps track of bundle directories and symlinks to enable partial rebuilding. 51 ContentChanges *contentChangeMap 52 53 // If enabled, keeps a revision map for all content. 54 gitInfo *gitInfo 55 } 56 57 func (h *HugoSites) IsMultihost() bool { 58 return h != nil && h.multihost 59 } 60 61 func (h *HugoSites) NumLogErrors() int { 62 if h == nil { 63 return 0 64 } 65 return int(h.Log.LogCountForLevelsGreaterThanorEqualTo(jww.LevelError)) 66 } 67 68 func (h *HugoSites) PrintProcessingStats(w io.Writer) { 69 stats := make([]*helpers.ProcessingStats, len(h.Sites)) 70 for i := 0; i < len(h.Sites); i++ { 71 stats[i] = h.Sites[i].PathSpec.ProcessingStats 72 } 73 helpers.ProcessingStatsTable(w, stats...) 74 } 75 76 func (h *HugoSites) langSite() map[string]*Site { 77 m := make(map[string]*Site) 78 for _, s := range h.Sites { 79 m[s.Language.Lang] = s 80 } 81 return m 82 } 83 84 // GetContentPage finds a Page with content given the absolute filename. 85 // Returns nil if none found. 86 func (h *HugoSites) GetContentPage(filename string) *Page { 87 for _, s := range h.Sites { 88 pos := s.rawAllPages.findPagePosByFilename(filename) 89 if pos == -1 { 90 continue 91 } 92 return s.rawAllPages[pos] 93 } 94 95 // If not found already, this may be bundled in another content file. 96 dir := filepath.Dir(filename) 97 98 for _, s := range h.Sites { 99 pos := s.rawAllPages.findPagePosByFilnamePrefix(dir) 100 if pos == -1 { 101 continue 102 } 103 return s.rawAllPages[pos] 104 } 105 return nil 106 } 107 108 // NewHugoSites creates a new collection of sites given the input sites, building 109 // a language configuration based on those. 110 func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { 111 112 if cfg.Language != nil { 113 return nil, errors.New("Cannot provide Language in Cfg when sites are provided") 114 } 115 116 langConfig, err := newMultiLingualFromSites(cfg.Cfg, sites...) 117 118 if err != nil { 119 return nil, err 120 } 121 122 var contentChangeTracker *contentChangeMap 123 124 h := &HugoSites{ 125 running: cfg.Running, 126 multilingual: langConfig, 127 multihost: cfg.Cfg.GetBool("multihost"), 128 Sites: sites} 129 130 for _, s := range sites { 131 s.owner = h 132 } 133 134 if err := applyDepsIfNeeded(cfg, sites...); err != nil { 135 return nil, err 136 } 137 138 h.Deps = sites[0].Deps 139 140 // Only needed in server mode. 141 // TODO(bep) clean up the running vs watching terms 142 if cfg.Running { 143 contentChangeTracker = &contentChangeMap{pathSpec: h.PathSpec, symContent: make(map[string]map[string]bool)} 144 h.ContentChanges = contentChangeTracker 145 } 146 147 if err := h.initGitInfo(); err != nil { 148 return nil, err 149 } 150 151 return h, nil 152 } 153 154 func (h *HugoSites) initGitInfo() error { 155 if h.Cfg.GetBool("enableGitInfo") { 156 gi, err := newGitInfo(h.Cfg) 157 if err != nil { 158 h.Log.ERROR.Println("Failed to read Git log:", err) 159 } else { 160 h.gitInfo = gi 161 } 162 } 163 return nil 164 } 165 166 func applyDepsIfNeeded(cfg deps.DepsCfg, sites ...*Site) error { 167 if cfg.TemplateProvider == nil { 168 cfg.TemplateProvider = tplimpl.DefaultTemplateProvider 169 } 170 171 if cfg.TranslationProvider == nil { 172 cfg.TranslationProvider = i18n.NewTranslationProvider() 173 } 174 175 var ( 176 d *deps.Deps 177 err error 178 ) 179 180 for _, s := range sites { 181 if s.Deps != nil { 182 continue 183 } 184 185 if d == nil { 186 cfg.Language = s.Language 187 cfg.WithTemplate = s.withSiteTemplates(cfg.WithTemplate) 188 189 var err error 190 d, err = deps.New(cfg) 191 if err != nil { 192 return err 193 } 194 195 d.OutputFormatsConfig = s.outputFormatsConfig 196 s.Deps = d 197 198 if err = d.LoadResources(); err != nil { 199 return err 200 } 201 202 } else { 203 d, err = d.ForLanguage(s.Language) 204 if err != nil { 205 return err 206 } 207 d.OutputFormatsConfig = s.outputFormatsConfig 208 s.Deps = d 209 } 210 211 s.resourceSpec, err = resource.NewSpec(s.Deps.PathSpec, s.mediaTypesConfig) 212 if err != nil { 213 return err 214 } 215 216 } 217 218 return nil 219 } 220 221 // NewHugoSites creates HugoSites from the given config. 222 func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { 223 sites, err := createSitesFromConfig(cfg) 224 if err != nil { 225 return nil, err 226 } 227 return newHugoSites(cfg, sites...) 228 } 229 230 func (s *Site) withSiteTemplates(withTemplates ...func(templ tpl.TemplateHandler) error) func(templ tpl.TemplateHandler) error { 231 return func(templ tpl.TemplateHandler) error { 232 templ.LoadTemplates("") 233 234 for _, wt := range withTemplates { 235 if wt == nil { 236 continue 237 } 238 if err := wt(templ); err != nil { 239 return err 240 } 241 } 242 243 return nil 244 } 245 } 246 247 func createSitesFromConfig(cfg deps.DepsCfg) ([]*Site, error) { 248 249 var ( 250 sites []*Site 251 ) 252 253 languages := getLanguages(cfg.Cfg) 254 255 for _, lang := range languages { 256 if lang.Disabled { 257 continue 258 } 259 var s *Site 260 var err error 261 cfg.Language = lang 262 s, err = newSite(cfg) 263 264 if err != nil { 265 return nil, err 266 } 267 268 sites = append(sites, s) 269 } 270 271 return sites, nil 272 } 273 274 // Reset resets the sites and template caches, making it ready for a full rebuild. 275 func (h *HugoSites) reset() { 276 for i, s := range h.Sites { 277 h.Sites[i] = s.reset() 278 } 279 } 280 281 // resetLogs resets the log counters etc. Used to do a new build on the same sites. 282 func (h *HugoSites) resetLogs() { 283 h.Log.ResetLogCounters() 284 for _, s := range h.Sites { 285 s.Deps.DistinctErrorLog = helpers.NewDistinctLogger(h.Log.ERROR) 286 } 287 } 288 289 func (h *HugoSites) createSitesFromConfig() error { 290 oldLangs, _ := h.Cfg.Get("languagesSorted").(langs.Languages) 291 292 if err := loadLanguageSettings(h.Cfg, oldLangs); err != nil { 293 return err 294 } 295 296 depsCfg := deps.DepsCfg{Fs: h.Fs, Cfg: h.Cfg} 297 298 sites, err := createSitesFromConfig(depsCfg) 299 300 if err != nil { 301 return err 302 } 303 304 langConfig, err := newMultiLingualFromSites(depsCfg.Cfg, sites...) 305 306 if err != nil { 307 return err 308 } 309 310 h.Sites = sites 311 312 for _, s := range sites { 313 s.owner = h 314 } 315 316 if err := applyDepsIfNeeded(depsCfg, sites...); err != nil { 317 return err 318 } 319 320 h.Deps = sites[0].Deps 321 322 h.multilingual = langConfig 323 h.multihost = h.Deps.Cfg.GetBool("multihost") 324 325 return nil 326 } 327 328 func (h *HugoSites) toSiteInfos() []*SiteInfo { 329 infos := make([]*SiteInfo, len(h.Sites)) 330 for i, s := range h.Sites { 331 infos[i] = &s.Info 332 } 333 return infos 334 } 335 336 // BuildCfg holds build options used to, as an example, skip the render step. 337 type BuildCfg struct { 338 // Reset site state before build. Use to force full rebuilds. 339 ResetState bool 340 // Re-creates the sites from configuration before a build. 341 // This is needed if new languages are added. 342 CreateSitesFromConfig bool 343 // Skip rendering. Useful for testing. 344 SkipRender bool 345 // Use this to indicate what changed (for rebuilds). 346 whatChanged *whatChanged 347 // Recently visited URLs. This is used for partial re-rendering. 348 RecentlyVisited map[string]bool 349 } 350 351 // shouldRender is used in the Fast Render Mode to determine if we need to re-render 352 // a Page: If it is recently visited (the home pages will always be in this set) or changed. 353 // Note that a page does not have to have a content page / file. 354 // For regular builds, this will allways return true. 355 func (cfg *BuildCfg) shouldRender(p *Page) bool { 356 if len(cfg.RecentlyVisited) == 0 { 357 return true 358 } 359 360 if cfg.RecentlyVisited[p.RelPermalink()] { 361 return true 362 } 363 364 if cfg.whatChanged != nil && p.File != nil { 365 return cfg.whatChanged.files[p.File.Filename()] 366 } 367 368 return false 369 } 370 371 func (h *HugoSites) renderCrossSitesArtifacts() error { 372 373 if !h.multilingual.enabled() || h.IsMultihost() { 374 return nil 375 } 376 377 sitemapEnabled := false 378 for _, s := range h.Sites { 379 if s.isEnabled(kindSitemap) { 380 sitemapEnabled = true 381 break 382 } 383 } 384 385 if !sitemapEnabled { 386 return nil 387 } 388 389 // TODO(bep) DRY 390 sitemapDefault := parseSitemap(h.Cfg.GetStringMap("sitemap")) 391 392 s := h.Sites[0] 393 394 smLayouts := []string{"sitemapindex.xml", "_default/sitemapindex.xml", "_internal/_default/sitemapindex.xml"} 395 396 return s.renderAndWriteXML(&s.PathSpec.ProcessingStats.Sitemaps, "sitemapindex", 397 sitemapDefault.Filename, h.toSiteInfos(), s.appendThemeTemplates(smLayouts)...) 398 } 399 400 func (h *HugoSites) assignMissingTranslations() error { 401 402 // This looks heavy, but it should be a small number of nodes by now. 403 allPages := h.findAllPagesByKindNotIn(KindPage) 404 for _, nodeType := range []string{KindHome, KindSection, KindTaxonomy, KindTaxonomyTerm} { 405 nodes := h.findPagesByKindIn(nodeType, allPages) 406 407 // Assign translations 408 for _, t1 := range nodes { 409 for _, t2 := range nodes { 410 if t1.isNewTranslation(t2) { 411 t1.translations = append(t1.translations, t2) 412 } 413 } 414 } 415 } 416 417 // Now we can sort the translations. 418 for _, p := range allPages { 419 if len(p.translations) > 0 { 420 pageBy(languagePageSort).Sort(p.translations) 421 } 422 } 423 return nil 424 425 } 426 427 // createMissingPages creates home page, taxonomies etc. that isnt't created as an 428 // effect of having a content file. 429 func (h *HugoSites) createMissingPages() error { 430 var newPages Pages 431 432 for _, s := range h.Sites { 433 if s.isEnabled(KindHome) { 434 // home pages 435 home := s.findPagesByKind(KindHome) 436 if len(home) > 1 { 437 panic("Too many homes") 438 } 439 if len(home) == 0 { 440 n := s.newHomePage() 441 s.Pages = append(s.Pages, n) 442 newPages = append(newPages, n) 443 } 444 } 445 446 // Will create content-less root sections. 447 newSections := s.assembleSections() 448 s.Pages = append(s.Pages, newSections...) 449 newPages = append(newPages, newSections...) 450 451 // taxonomy list and terms pages 452 taxonomies := s.Language.GetStringMapString("taxonomies") 453 if len(taxonomies) > 0 { 454 taxonomyPages := s.findPagesByKind(KindTaxonomy) 455 taxonomyTermsPages := s.findPagesByKind(KindTaxonomyTerm) 456 for _, plural := range taxonomies { 457 if s.isEnabled(KindTaxonomyTerm) { 458 foundTaxonomyTermsPage := false 459 for _, p := range taxonomyTermsPages { 460 if p.sections[0] == plural { 461 foundTaxonomyTermsPage = true 462 break 463 } 464 } 465 466 if !foundTaxonomyTermsPage { 467 n := s.newTaxonomyTermsPage(plural) 468 s.Pages = append(s.Pages, n) 469 newPages = append(newPages, n) 470 } 471 } 472 473 if s.isEnabled(KindTaxonomy) { 474 for key := range s.Taxonomies[plural] { 475 foundTaxonomyPage := false 476 origKey := key 477 478 if s.Info.preserveTaxonomyNames { 479 key = s.PathSpec.MakePathSanitized(key) 480 } 481 for _, p := range taxonomyPages { 482 // Some people may have /authors/MaxMustermann etc. as paths. 483 // p.sections contains the raw values from the file system. 484 // See https://github.com/gohugoio/hugo/issues/4238 485 singularKey := s.PathSpec.MakePathSanitized(p.sections[1]) 486 if p.sections[0] == plural && singularKey == key { 487 foundTaxonomyPage = true 488 break 489 } 490 } 491 492 if !foundTaxonomyPage { 493 n := s.newTaxonomyPage(plural, origKey) 494 s.Pages = append(s.Pages, n) 495 newPages = append(newPages, n) 496 } 497 } 498 } 499 } 500 } 501 } 502 503 if len(newPages) > 0 { 504 // This resorting is unfortunate, but it also needs to be sorted 505 // when sections are created. 506 first := h.Sites[0] 507 508 first.AllPages = append(first.AllPages, newPages...) 509 510 first.AllPages.Sort() 511 512 for _, s := range h.Sites { 513 s.Pages.Sort() 514 } 515 516 for i := 1; i < len(h.Sites); i++ { 517 h.Sites[i].AllPages = first.AllPages 518 } 519 } 520 521 return nil 522 } 523 524 func (h *HugoSites) removePageByFilename(filename string) { 525 for _, s := range h.Sites { 526 s.removePageFilename(filename) 527 } 528 } 529 530 func (h *HugoSites) setupTranslations() { 531 for _, s := range h.Sites { 532 for _, p := range s.rawAllPages { 533 if p.Kind == kindUnknown { 534 p.Kind = p.s.kindFromSections(p.sections) 535 } 536 537 if !p.s.isEnabled(p.Kind) { 538 continue 539 } 540 541 shouldBuild := p.shouldBuild() 542 s.updateBuildStats(p) 543 if shouldBuild { 544 if p.headless { 545 s.headlessPages = append(s.headlessPages, p) 546 } else { 547 s.Pages = append(s.Pages, p) 548 } 549 } 550 } 551 } 552 553 allPages := make(Pages, 0) 554 555 for _, s := range h.Sites { 556 allPages = append(allPages, s.Pages...) 557 } 558 559 allPages.Sort() 560 561 for _, s := range h.Sites { 562 s.AllPages = allPages 563 } 564 565 // Pull over the collections from the master site 566 for i := 1; i < len(h.Sites); i++ { 567 h.Sites[i].Data = h.Sites[0].Data 568 } 569 570 if len(h.Sites) > 1 { 571 allTranslations := pagesToTranslationsMap(allPages) 572 assignTranslationsToPages(allTranslations, allPages) 573 } 574 } 575 576 func (s *Site) preparePagesForRender(start bool) error { 577 for _, p := range s.Pages { 578 p.setContentInit(start) 579 if err := p.initMainOutputFormat(); err != nil { 580 return err 581 } 582 } 583 584 for _, p := range s.headlessPages { 585 p.setContentInit(start) 586 if err := p.initMainOutputFormat(); err != nil { 587 return err 588 } 589 } 590 591 return nil 592 } 593 594 // Pages returns all pages for all sites. 595 func (h *HugoSites) Pages() Pages { 596 return h.Sites[0].AllPages 597 } 598 599 func handleShortcodes(p *PageWithoutContent, rawContentCopy []byte) ([]byte, error) { 600 if p.shortcodeState != nil && p.shortcodeState.contentShortcodes.Len() > 0 { 601 p.s.Log.DEBUG.Printf("Replace %d shortcodes in %q", p.shortcodeState.contentShortcodes.Len(), p.BaseFileName()) 602 err := p.shortcodeState.executeShortcodesForDelta(p) 603 604 if err != nil { 605 return rawContentCopy, err 606 } 607 608 rawContentCopy, err = replaceShortcodeTokens(rawContentCopy, shortcodePlaceholderPrefix, p.shortcodeState.renderedShortcodes) 609 610 if err != nil { 611 p.s.Log.FATAL.Printf("Failed to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error()) 612 } 613 } 614 615 return rawContentCopy, nil 616 } 617 618 func (s *Site) updateBuildStats(page *Page) { 619 if page.IsDraft() { 620 s.draftCount++ 621 } 622 623 if page.IsFuture() { 624 s.futureCount++ 625 } 626 627 if page.IsExpired() { 628 s.expiredCount++ 629 } 630 } 631 632 func (h *HugoSites) findPagesByKindNotIn(kind string, inPages Pages) Pages { 633 return h.Sites[0].findPagesByKindNotIn(kind, inPages) 634 } 635 636 func (h *HugoSites) findPagesByKindIn(kind string, inPages Pages) Pages { 637 return h.Sites[0].findPagesByKindIn(kind, inPages) 638 } 639 640 func (h *HugoSites) findAllPagesByKind(kind string) Pages { 641 return h.findPagesByKindIn(kind, h.Sites[0].AllPages) 642 } 643 644 func (h *HugoSites) findAllPagesByKindNotIn(kind string) Pages { 645 return h.findPagesByKindNotIn(kind, h.Sites[0].AllPages) 646 } 647 648 func (h *HugoSites) findPagesByShortcode(shortcode string) Pages { 649 var pages Pages 650 for _, s := range h.Sites { 651 pages = append(pages, s.findPagesByShortcode(shortcode)...) 652 } 653 return pages 654 } 655 656 // Used in partial reloading to determine if the change is in a bundle. 657 type contentChangeMap struct { 658 mu sync.RWMutex 659 branches []string 660 leafs []string 661 662 pathSpec *helpers.PathSpec 663 664 // Hugo supports symlinked content (both directories and files). This 665 // can lead to situations where the same file can be referenced from several 666 // locations in /content -- which is really cool, but also means we have to 667 // go an extra mile to handle changes. 668 // This map is only used in watch mode. 669 // It maps either file to files or the real dir to a set of content directories where it is in use. 670 symContent map[string]map[string]bool 671 symContentMu sync.Mutex 672 } 673 674 func (m *contentChangeMap) add(filename string, tp bundleDirType) { 675 m.mu.Lock() 676 dir := filepath.Dir(filename) + helpers.FilePathSeparator 677 dir = strings.TrimPrefix(dir, ".") 678 switch tp { 679 case bundleBranch: 680 m.branches = append(m.branches, dir) 681 case bundleLeaf: 682 m.leafs = append(m.leafs, dir) 683 default: 684 panic("invalid bundle type") 685 } 686 m.mu.Unlock() 687 } 688 689 // Track the addition of bundle dirs. 690 func (m *contentChangeMap) handleBundles(b *bundleDirs) { 691 for _, bd := range b.bundles { 692 m.add(bd.fi.Path(), bd.tp) 693 } 694 } 695 696 // resolveAndRemove resolves the given filename to the root folder of a bundle, if relevant. 697 // It also removes the entry from the map. It will be re-added again by the partial 698 // build if it still is a bundle. 699 func (m *contentChangeMap) resolveAndRemove(filename string) (string, string, bundleDirType) { 700 m.mu.RLock() 701 defer m.mu.RUnlock() 702 703 // Bundles share resources, so we need to start from the virtual root. 704 relPath, _ := m.pathSpec.RelContentDir(filename) 705 dir, name := filepath.Split(relPath) 706 if !strings.HasSuffix(dir, helpers.FilePathSeparator) { 707 dir += helpers.FilePathSeparator 708 } 709 710 fileTp, isContent := classifyBundledFile(name) 711 712 // This may be a member of a bundle. Start with branch bundles, the most specific. 713 if fileTp == bundleBranch || (fileTp == bundleNot && !isContent) { 714 for i, b := range m.branches { 715 if b == dir { 716 m.branches = append(m.branches[:i], m.branches[i+1:]...) 717 return dir, b, bundleBranch 718 } 719 } 720 } 721 722 // And finally the leaf bundles, which can contain anything. 723 for i, l := range m.leafs { 724 if strings.HasPrefix(dir, l) { 725 m.leafs = append(m.leafs[:i], m.leafs[i+1:]...) 726 return dir, l, bundleLeaf 727 } 728 } 729 730 // Not part of any bundle 731 return dir, filename, bundleNot 732 } 733 734 func (m *contentChangeMap) addSymbolicLinkMapping(from, to string) { 735 m.symContentMu.Lock() 736 mm, found := m.symContent[from] 737 if !found { 738 mm = make(map[string]bool) 739 m.symContent[from] = mm 740 } 741 mm[to] = true 742 m.symContentMu.Unlock() 743 } 744 745 func (m *contentChangeMap) GetSymbolicLinkMappings(dir string) []string { 746 mm, found := m.symContent[dir] 747 if !found { 748 return nil 749 } 750 dirs := make([]string, len(mm)) 751 i := 0 752 for dir := range mm { 753 dirs[i] = dir 754 i++ 755 } 756 757 sort.Strings(dirs) 758 return dirs 759 }