github.com/rabbouni145/gg@v0.47.1/hugolib/site.go (about) 1 // Copyright 2017 The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package hugolib 15 16 import ( 17 "errors" 18 "fmt" 19 "html/template" 20 "io" 21 "log" 22 "mime" 23 "net/url" 24 "os" 25 "path/filepath" 26 "sort" 27 "strconv" 28 "strings" 29 "time" 30 31 "github.com/gohugoio/hugo/common/maps" 32 "github.com/gohugoio/hugo/publisher" 33 "github.com/gohugoio/hugo/resource" 34 35 "github.com/gohugoio/hugo/langs" 36 37 src "github.com/gohugoio/hugo/source" 38 39 "golang.org/x/sync/errgroup" 40 41 "github.com/gohugoio/hugo/config" 42 43 "github.com/gohugoio/hugo/media" 44 45 "github.com/markbates/inflect" 46 "golang.org/x/net/context" 47 48 "github.com/fsnotify/fsnotify" 49 bp "github.com/gohugoio/hugo/bufferpool" 50 "github.com/gohugoio/hugo/deps" 51 "github.com/gohugoio/hugo/helpers" 52 "github.com/gohugoio/hugo/hugolib/pagemeta" 53 "github.com/gohugoio/hugo/output" 54 "github.com/gohugoio/hugo/parser" 55 "github.com/gohugoio/hugo/related" 56 "github.com/gohugoio/hugo/source" 57 "github.com/gohugoio/hugo/tpl" 58 "github.com/spf13/afero" 59 "github.com/spf13/cast" 60 "github.com/spf13/nitro" 61 "github.com/spf13/viper" 62 ) 63 64 // used to indicate if run as a test. 65 var testMode bool 66 67 var defaultTimer *nitro.B 68 69 // Site contains all the information relevant for constructing a static 70 // site. The basic flow of information is as follows: 71 // 72 // 1. A list of Files is parsed and then converted into Pages. 73 // 74 // 2. Pages contain sections (based on the file they were generated from), 75 // aliases and slugs (included in a pages frontmatter) which are the 76 // various targets that will get generated. There will be canonical 77 // listing. The canonical path can be overruled based on a pattern. 78 // 79 // 3. Taxonomies are created via configuration and will present some aspect of 80 // the final page and typically a perm url. 81 // 82 // 4. All Pages are passed through a template based on their desired 83 // layout based on numerous different elements. 84 // 85 // 5. The entire collection of files is written to disk. 86 type Site struct { 87 owner *HugoSites 88 89 *PageCollections 90 91 Taxonomies TaxonomyList 92 93 // Plural is what we get in the folder, so keep track of this mapping 94 // to get the singular form from that value. 95 taxonomiesPluralSingular map[string]string 96 97 // This is temporary, see https://github.com/gohugoio/hugo/issues/2835 98 // Maps "actors-gerard-depardieu" to "Gérard Depardieu" when preserveTaxonomyNames 99 // is set. 100 taxonomiesOrigKey map[string]string 101 102 Sections Taxonomy 103 Info SiteInfo 104 Menus Menus 105 timer *nitro.B 106 107 layoutHandler *output.LayoutHandler 108 109 draftCount int 110 futureCount int 111 expiredCount int 112 113 Data map[string]interface{} 114 Language *langs.Language 115 116 disabledKinds map[string]bool 117 118 // Output formats defined in site config per Page Kind, or some defaults 119 // if not set. 120 // Output formats defined in Page front matter will override these. 121 outputFormats map[string]output.Formats 122 123 // All the output formats and media types available for this site. 124 // These values will be merged from the Hugo defaults, the site config and, 125 // finally, the language settings. 126 outputFormatsConfig output.Formats 127 mediaTypesConfig media.Types 128 129 siteConfig SiteConfig 130 131 // How to handle page front matter. 132 frontmatterHandler pagemeta.FrontMatterHandler 133 134 // We render each site for all the relevant output formats in serial with 135 // this rendering context pointing to the current one. 136 rc *siteRenderingContext 137 138 // The output formats that we need to render this site in. This slice 139 // will be fixed once set. 140 // This will be the union of Site.Pages' outputFormats. 141 // This slice will be sorted. 142 renderFormats output.Formats 143 144 // Logger etc. 145 *deps.Deps `json:"-"` 146 147 // The func used to title case titles. 148 titleFunc func(s string) string 149 150 relatedDocsHandler *relatedDocsHandler 151 siteRefLinker 152 153 publisher publisher.Publisher 154 } 155 156 type siteRenderingContext struct { 157 output.Format 158 } 159 160 func (s *Site) initRenderFormats() { 161 formatSet := make(map[string]bool) 162 formats := output.Formats{} 163 for _, p := range s.Pages { 164 for _, f := range p.outputFormats { 165 if !formatSet[f.Name] { 166 formats = append(formats, f) 167 formatSet[f.Name] = true 168 } 169 } 170 } 171 172 sort.Sort(formats) 173 s.renderFormats = formats 174 } 175 176 func (s *Site) isEnabled(kind string) bool { 177 if kind == kindUnknown { 178 panic("Unknown kind") 179 } 180 return !s.disabledKinds[kind] 181 } 182 183 // reset returns a new Site prepared for rebuild. 184 func (s *Site) reset() *Site { 185 return &Site{Deps: s.Deps, 186 layoutHandler: output.NewLayoutHandler(), 187 disabledKinds: s.disabledKinds, 188 titleFunc: s.titleFunc, 189 relatedDocsHandler: newSearchIndexHandler(s.relatedDocsHandler.cfg), 190 siteRefLinker: s.siteRefLinker, 191 outputFormats: s.outputFormats, 192 rc: s.rc, 193 outputFormatsConfig: s.outputFormatsConfig, 194 frontmatterHandler: s.frontmatterHandler, 195 mediaTypesConfig: s.mediaTypesConfig, 196 Language: s.Language, 197 owner: s.owner, 198 publisher: s.publisher, 199 siteConfig: s.siteConfig, 200 PageCollections: newPageCollections()} 201 202 } 203 204 // newSite creates a new site with the given configuration. 205 func newSite(cfg deps.DepsCfg) (*Site, error) { 206 c := newPageCollections() 207 208 if cfg.Language == nil { 209 cfg.Language = langs.NewDefaultLanguage(cfg.Cfg) 210 } 211 212 disabledKinds := make(map[string]bool) 213 for _, disabled := range cast.ToStringSlice(cfg.Language.Get("disableKinds")) { 214 disabledKinds[disabled] = true 215 } 216 217 var ( 218 mediaTypesConfig []map[string]interface{} 219 outputFormatsConfig []map[string]interface{} 220 221 siteOutputFormatsConfig output.Formats 222 siteMediaTypesConfig media.Types 223 err error 224 ) 225 226 // Add language last, if set, so it gets precedence. 227 for _, cfg := range []config.Provider{cfg.Cfg, cfg.Language} { 228 if cfg.IsSet("mediaTypes") { 229 mediaTypesConfig = append(mediaTypesConfig, cfg.GetStringMap("mediaTypes")) 230 } 231 if cfg.IsSet("outputFormats") { 232 outputFormatsConfig = append(outputFormatsConfig, cfg.GetStringMap("outputFormats")) 233 } 234 } 235 236 siteMediaTypesConfig, err = media.DecodeTypes(mediaTypesConfig...) 237 if err != nil { 238 return nil, err 239 } 240 241 siteOutputFormatsConfig, err = output.DecodeFormats(siteMediaTypesConfig, outputFormatsConfig...) 242 if err != nil { 243 return nil, err 244 } 245 246 outputFormats, err := createSiteOutputFormats(siteOutputFormatsConfig, cfg.Language) 247 if err != nil { 248 return nil, err 249 } 250 251 var relatedContentConfig related.Config 252 253 if cfg.Language.IsSet("related") { 254 relatedContentConfig, err = related.DecodeConfig(cfg.Language.Get("related")) 255 if err != nil { 256 return nil, err 257 } 258 } else { 259 relatedContentConfig = related.DefaultConfig 260 taxonomies := cfg.Language.GetStringMapString("taxonomies") 261 if _, found := taxonomies["tag"]; found { 262 relatedContentConfig.Add(related.IndexConfig{Name: "tags", Weight: 80}) 263 } 264 } 265 266 titleFunc := helpers.GetTitleFunc(cfg.Language.GetString("titleCaseStyle")) 267 268 frontMatterHandler, err := pagemeta.NewFrontmatterHandler(cfg.Logger, cfg.Cfg) 269 if err != nil { 270 return nil, err 271 } 272 273 s := &Site{ 274 PageCollections: c, 275 layoutHandler: output.NewLayoutHandler(), 276 Language: cfg.Language, 277 disabledKinds: disabledKinds, 278 titleFunc: titleFunc, 279 relatedDocsHandler: newSearchIndexHandler(relatedContentConfig), 280 outputFormats: outputFormats, 281 rc: &siteRenderingContext{output.HTMLFormat}, 282 outputFormatsConfig: siteOutputFormatsConfig, 283 mediaTypesConfig: siteMediaTypesConfig, 284 frontmatterHandler: frontMatterHandler, 285 } 286 287 return s, nil 288 289 } 290 291 // NewSite creates a new site with the given dependency configuration. 292 // The site will have a template system loaded and ready to use. 293 // Note: This is mainly used in single site tests. 294 func NewSite(cfg deps.DepsCfg) (*Site, error) { 295 s, err := newSite(cfg) 296 if err != nil { 297 return nil, err 298 } 299 300 if err = applyDeps(cfg, s); err != nil { 301 return nil, err 302 } 303 304 return s, nil 305 } 306 307 // NewSiteDefaultLang creates a new site in the default language. 308 // The site will have a template system loaded and ready to use. 309 // Note: This is mainly used in single site tests. 310 // TODO(bep) test refactor -- remove 311 func NewSiteDefaultLang(withTemplate ...func(templ tpl.TemplateHandler) error) (*Site, error) { 312 v := viper.New() 313 if err := loadDefaultSettingsFor(v); err != nil { 314 return nil, err 315 } 316 return newSiteForLang(langs.NewDefaultLanguage(v), withTemplate...) 317 } 318 319 // NewEnglishSite creates a new site in English language. 320 // The site will have a template system loaded and ready to use. 321 // Note: This is mainly used in single site tests. 322 // TODO(bep) test refactor -- remove 323 func NewEnglishSite(withTemplate ...func(templ tpl.TemplateHandler) error) (*Site, error) { 324 v := viper.New() 325 if err := loadDefaultSettingsFor(v); err != nil { 326 return nil, err 327 } 328 return newSiteForLang(langs.NewLanguage("en", v), withTemplate...) 329 } 330 331 // newSiteForLang creates a new site in the given language. 332 func newSiteForLang(lang *langs.Language, withTemplate ...func(templ tpl.TemplateHandler) error) (*Site, error) { 333 withTemplates := func(templ tpl.TemplateHandler) error { 334 for _, wt := range withTemplate { 335 if err := wt(templ); err != nil { 336 return err 337 } 338 } 339 return nil 340 } 341 342 cfg := deps.DepsCfg{WithTemplate: withTemplates, Cfg: lang} 343 344 return NewSiteForCfg(cfg) 345 346 } 347 348 // NewSiteForCfg creates a new site for the given configuration. 349 // The site will have a template system loaded and ready to use. 350 // Note: This is mainly used in single site tests. 351 func NewSiteForCfg(cfg deps.DepsCfg) (*Site, error) { 352 h, err := NewHugoSites(cfg) 353 if err != nil { 354 return nil, err 355 } 356 return h.Sites[0], nil 357 358 } 359 360 type SiteInfos []*SiteInfo 361 362 // First is a convenience method to get the first Site, i.e. the main language. 363 func (s SiteInfos) First() *SiteInfo { 364 if len(s) == 0 { 365 return nil 366 } 367 return s[0] 368 } 369 370 type SiteInfo struct { 371 Taxonomies TaxonomyList 372 Authors AuthorList 373 Social SiteSocial 374 *PageCollections 375 Menus *Menus 376 Hugo *HugoInfo 377 Title string 378 RSSLink string 379 Author map[string]interface{} 380 LanguageCode string 381 Copyright string 382 LastChange time.Time 383 Permalinks PermalinkOverrides 384 Params map[string]interface{} 385 BuildDrafts bool 386 canonifyURLs bool 387 relativeURLs bool 388 uglyURLs func(p *Page) bool 389 preserveTaxonomyNames bool 390 Data *map[string]interface{} 391 owner *HugoSites 392 s *Site 393 Language *langs.Language 394 LanguagePrefix string 395 Languages langs.Languages 396 defaultContentLanguageInSubdir bool 397 sectionPagesMenu string 398 } 399 400 func (s *SiteInfo) Config() SiteConfig { 401 return s.s.siteConfig 402 } 403 404 func (s *SiteInfo) String() string { 405 return fmt.Sprintf("Site(%q)", s.Title) 406 } 407 408 func (s *SiteInfo) BaseURL() template.URL { 409 return template.URL(s.s.PathSpec.BaseURL.String()) 410 } 411 412 // ServerPort returns the port part of the BaseURL, 0 if none found. 413 func (s *SiteInfo) ServerPort() int { 414 ps := s.s.PathSpec.BaseURL.URL().Port() 415 if ps == "" { 416 return 0 417 } 418 p, err := strconv.Atoi(ps) 419 if err != nil { 420 return 0 421 } 422 return p 423 } 424 425 // GoogleAnalytics is kept here for historic reasons. 426 func (s *SiteInfo) GoogleAnalytics() string { 427 return s.Config().Services.GoogleAnalytics.ID 428 429 } 430 431 // DisqusShortname is kept here for historic reasons. 432 func (s *SiteInfo) DisqusShortname() string { 433 return s.Config().Services.Disqus.Shortname 434 } 435 436 // SiteSocial is a place to put social details on a site level. These are the 437 // standard keys that themes will expect to have available, but can be 438 // expanded to any others on a per site basis 439 // github 440 // facebook 441 // facebook_admin 442 // twitter 443 // twitter_domain 444 // googleplus 445 // pinterest 446 // instagram 447 // youtube 448 // linkedin 449 type SiteSocial map[string]string 450 451 // Param is a convenience method to do lookups in SiteInfo's Params map. 452 // 453 // This method is also implemented on Page and Node. 454 func (s *SiteInfo) Param(key interface{}) (interface{}, error) { 455 keyStr, err := cast.ToStringE(key) 456 if err != nil { 457 return nil, err 458 } 459 keyStr = strings.ToLower(keyStr) 460 return s.Params[keyStr], nil 461 } 462 463 func (s *SiteInfo) IsMultiLingual() bool { 464 return len(s.Languages) > 1 465 } 466 467 func (s *SiteInfo) IsServer() bool { 468 return s.owner.running 469 } 470 471 type siteRefLinker struct { 472 s *Site 473 474 errorLogger *log.Logger 475 notFoundURL string 476 } 477 478 func newSiteRefLinker(cfg config.Provider, s *Site) (siteRefLinker, error) { 479 logger := s.Log.ERROR 480 481 notFoundURL := cfg.GetString("refLinksNotFoundURL") 482 errLevel := cfg.GetString("refLinksErrorLevel") 483 if strings.EqualFold(errLevel, "warning") { 484 logger = s.Log.WARN 485 } 486 return siteRefLinker{s: s, errorLogger: logger, notFoundURL: notFoundURL}, nil 487 } 488 489 func (s siteRefLinker) logNotFound(ref, what string, p *Page) { 490 if p != nil { 491 s.errorLogger.Printf("REF_NOT_FOUND: Ref %q: %s", ref, what) 492 } else { 493 s.errorLogger.Printf("REF_NOT_FOUND: Ref %q from page %q: %s", ref, p.absoluteSourceRef(), what) 494 } 495 496 } 497 498 func (s *siteRefLinker) refLink(ref string, page *Page, relative bool, outputFormat string) (string, error) { 499 500 var refURL *url.URL 501 var err error 502 503 ref = filepath.ToSlash(ref) 504 505 refURL, err = url.Parse(ref) 506 507 if err != nil { 508 return s.notFoundURL, err 509 } 510 511 var target *Page 512 var link string 513 514 if refURL.Path != "" { 515 target, err := s.s.getPageNew(page, refURL.Path) 516 517 if err != nil { 518 s.logNotFound(refURL.Path, err.Error(), page) 519 return s.notFoundURL, nil 520 } 521 522 if target == nil { 523 s.logNotFound(refURL.Path, "page not found", page) 524 return s.notFoundURL, nil 525 } 526 527 var permalinker Permalinker = target 528 529 if outputFormat != "" { 530 o := target.OutputFormats().Get(outputFormat) 531 532 if o == nil { 533 s.logNotFound(refURL.Path, fmt.Sprintf("output format %q", outputFormat), page) 534 return s.notFoundURL, nil 535 } 536 permalinker = o 537 } 538 539 if relative { 540 link = permalinker.RelPermalink() 541 } else { 542 link = permalinker.Permalink() 543 } 544 } 545 546 if refURL.Fragment != "" { 547 link = link + "#" + refURL.Fragment 548 549 if refURL.Path != "" && target != nil && !target.getRenderingConfig().PlainIDAnchors { 550 link = link + ":" + target.UniqueID() 551 } else if page != nil && !page.getRenderingConfig().PlainIDAnchors { 552 link = link + ":" + page.UniqueID() 553 } 554 } 555 556 return link, nil 557 } 558 559 // Ref will give an absolute URL to ref in the given Page. 560 func (s *SiteInfo) Ref(ref string, page *Page, options ...string) (string, error) { 561 outputFormat := "" 562 if len(options) > 0 { 563 outputFormat = options[0] 564 } 565 566 return s.s.refLink(ref, page, false, outputFormat) 567 } 568 569 // RelRef will give an relative URL to ref in the given Page. 570 func (s *SiteInfo) RelRef(ref string, page *Page, options ...string) (string, error) { 571 outputFormat := "" 572 if len(options) > 0 { 573 outputFormat = options[0] 574 } 575 576 return s.s.refLink(ref, page, true, outputFormat) 577 } 578 579 func (s *Site) running() bool { 580 return s.owner != nil && s.owner.running 581 } 582 583 func (s *Site) multilingual() *Multilingual { 584 return s.owner.multilingual 585 } 586 587 func init() { 588 defaultTimer = nitro.Initalize() 589 } 590 591 func (s *Site) timerStep(step string) { 592 if s.timer == nil { 593 s.timer = defaultTimer 594 } 595 s.timer.Step(step) 596 } 597 598 type whatChanged struct { 599 source bool 600 other bool 601 files map[string]bool 602 } 603 604 // RegisterMediaTypes will register the Site's media types in the mime 605 // package, so it will behave correctly with Hugo's built-in server. 606 func (s *Site) RegisterMediaTypes() { 607 for _, mt := range s.mediaTypesConfig { 608 for _, suffix := range mt.Suffixes { 609 _ = mime.AddExtensionType(mt.Delimiter+suffix, mt.Type()+"; charset=utf-8") 610 } 611 } 612 } 613 614 func (s *Site) filterFileEvents(events []fsnotify.Event) []fsnotify.Event { 615 var filtered []fsnotify.Event 616 seen := make(map[fsnotify.Event]bool) 617 618 for _, ev := range events { 619 // Avoid processing the same event twice. 620 if seen[ev] { 621 continue 622 } 623 seen[ev] = true 624 625 if s.SourceSpec.IgnoreFile(ev.Name) { 626 continue 627 } 628 629 // Throw away any directories 630 isRegular, err := s.SourceSpec.IsRegularSourceFile(ev.Name) 631 if err != nil && os.IsNotExist(err) && (ev.Op&fsnotify.Remove == fsnotify.Remove || ev.Op&fsnotify.Rename == fsnotify.Rename) { 632 // Force keep of event 633 isRegular = true 634 } 635 if !isRegular { 636 continue 637 } 638 639 filtered = append(filtered, ev) 640 } 641 642 return filtered 643 } 644 645 func (s *Site) translateFileEvents(events []fsnotify.Event) []fsnotify.Event { 646 var filtered []fsnotify.Event 647 648 eventMap := make(map[string][]fsnotify.Event) 649 650 // We often get a Remove etc. followed by a Create, a Create followed by a Write. 651 // Remove the superflous events to mage the update logic simpler. 652 for _, ev := range events { 653 eventMap[ev.Name] = append(eventMap[ev.Name], ev) 654 } 655 656 for _, ev := range events { 657 mapped := eventMap[ev.Name] 658 659 // Keep one 660 found := false 661 var kept fsnotify.Event 662 for i, ev2 := range mapped { 663 if i == 0 { 664 kept = ev2 665 } 666 667 if ev2.Op&fsnotify.Write == fsnotify.Write { 668 kept = ev2 669 found = true 670 } 671 672 if !found && ev2.Op&fsnotify.Create == fsnotify.Create { 673 kept = ev2 674 } 675 } 676 677 filtered = append(filtered, kept) 678 } 679 680 return filtered 681 } 682 683 // reBuild partially rebuilds a site given the filesystem events. 684 // It returns whetever the content source was changed. 685 // TODO(bep) clean up/rewrite this method. 686 func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { 687 688 events = s.filterFileEvents(events) 689 events = s.translateFileEvents(events) 690 691 s.Log.DEBUG.Printf("Rebuild for events %q", events) 692 693 h := s.owner 694 695 s.timerStep("initialize rebuild") 696 697 // First we need to determine what changed 698 699 var ( 700 sourceChanged = []fsnotify.Event{} 701 sourceReallyChanged = []fsnotify.Event{} 702 contentFilesChanged []string 703 tmplChanged = []fsnotify.Event{} 704 dataChanged = []fsnotify.Event{} 705 i18nChanged = []fsnotify.Event{} 706 shortcodesChanged = make(map[string]bool) 707 sourceFilesChanged = make(map[string]bool) 708 709 // prevent spamming the log on changes 710 logger = helpers.NewDistinctFeedbackLogger() 711 ) 712 713 cachePartitions := make([]string, len(events)) 714 715 for i, ev := range events { 716 cachePartitions[i] = resource.ResourceKeyPartition(ev.Name) 717 718 if s.isContentDirEvent(ev) { 719 logger.Println("Source changed", ev) 720 sourceChanged = append(sourceChanged, ev) 721 } 722 if s.isLayoutDirEvent(ev) { 723 logger.Println("Template changed", ev) 724 tmplChanged = append(tmplChanged, ev) 725 726 if strings.Contains(ev.Name, "shortcodes") { 727 clearIsInnerShortcodeCache() 728 shortcode := filepath.Base(ev.Name) 729 shortcode = strings.TrimSuffix(shortcode, filepath.Ext(shortcode)) 730 shortcodesChanged[shortcode] = true 731 } 732 } 733 if s.isDataDirEvent(ev) { 734 logger.Println("Data changed", ev) 735 dataChanged = append(dataChanged, ev) 736 } 737 if s.isI18nEvent(ev) { 738 logger.Println("i18n changed", ev) 739 i18nChanged = append(dataChanged, ev) 740 } 741 } 742 743 // These in memory resource caches will be rebuilt on demand. 744 for _, s := range s.owner.Sites { 745 s.ResourceSpec.ResourceCache.DeletePartitions(cachePartitions...) 746 } 747 748 if len(tmplChanged) > 0 || len(i18nChanged) > 0 { 749 sites := s.owner.Sites 750 first := sites[0] 751 752 // TOD(bep) globals clean 753 if err := first.Deps.LoadResources(); err != nil { 754 return whatChanged{}, err 755 } 756 757 s.TemplateHandler().PrintErrors() 758 759 for i := 1; i < len(sites); i++ { 760 site := sites[i] 761 var err error 762 depsCfg := deps.DepsCfg{ 763 Language: site.Language, 764 MediaTypes: site.mediaTypesConfig, 765 OutputFormats: site.outputFormatsConfig, 766 } 767 site.Deps, err = first.Deps.ForLanguage(depsCfg) 768 if err != nil { 769 return whatChanged{}, err 770 } 771 } 772 773 s.timerStep("template prep") 774 } 775 776 if len(dataChanged) > 0 { 777 if err := s.readDataFromSourceFS(); err != nil { 778 s.Log.ERROR.Println(err) 779 } 780 } 781 782 for _, ev := range sourceChanged { 783 removed := false 784 785 if ev.Op&fsnotify.Remove == fsnotify.Remove { 786 removed = true 787 } 788 789 // Some editors (Vim) sometimes issue only a Rename operation when writing an existing file 790 // Sometimes a rename operation means that file has been renamed other times it means 791 // it's been updated 792 if ev.Op&fsnotify.Rename == fsnotify.Rename { 793 // If the file is still on disk, it's only been updated, if it's not, it's been moved 794 if ex, err := afero.Exists(s.Fs.Source, ev.Name); !ex || err != nil { 795 removed = true 796 } 797 } 798 if removed && isContentFile(ev.Name) { 799 h.removePageByFilename(ev.Name) 800 } 801 802 sourceReallyChanged = append(sourceReallyChanged, ev) 803 sourceFilesChanged[ev.Name] = true 804 } 805 806 for shortcode := range shortcodesChanged { 807 // There are certain scenarios that, when a shortcode changes, 808 // it isn't sufficient to just rerender the already parsed shortcode. 809 // One example is if the user adds a new shortcode to the content file first, 810 // and then creates the shortcode on the file system. 811 // To handle these scenarios, we must do a full reprocessing of the 812 // pages that keeps a reference to the changed shortcode. 813 pagesWithShortcode := h.findPagesByShortcode(shortcode) 814 for _, p := range pagesWithShortcode { 815 contentFilesChanged = append(contentFilesChanged, p.File.Filename()) 816 } 817 } 818 819 if len(sourceReallyChanged) > 0 || len(contentFilesChanged) > 0 { 820 var filenamesChanged []string 821 for _, e := range sourceReallyChanged { 822 filenamesChanged = append(filenamesChanged, e.Name) 823 } 824 if len(contentFilesChanged) > 0 { 825 filenamesChanged = append(filenamesChanged, contentFilesChanged...) 826 } 827 828 filenamesChanged = helpers.UniqueStrings(filenamesChanged) 829 830 if err := s.readAndProcessContent(filenamesChanged...); err != nil { 831 return whatChanged{}, err 832 } 833 834 } 835 836 changed := whatChanged{ 837 source: len(sourceChanged) > 0 || len(shortcodesChanged) > 0, 838 other: len(tmplChanged) > 0 || len(i18nChanged) > 0 || len(dataChanged) > 0, 839 files: sourceFilesChanged, 840 } 841 842 return changed, nil 843 844 } 845 846 func (s *Site) loadData(fs afero.Fs) (err error) { 847 spec := src.NewSourceSpec(s.PathSpec, fs) 848 fileSystem := spec.NewFilesystem("") 849 s.Data = make(map[string]interface{}) 850 for _, r := range fileSystem.Files() { 851 if err := s.handleDataFile(r); err != nil { 852 return err 853 } 854 } 855 856 return 857 } 858 859 func (s *Site) handleDataFile(r source.ReadableFile) error { 860 var current map[string]interface{} 861 862 f, err := r.Open() 863 if err != nil { 864 return fmt.Errorf("Failed to open data file %q: %s", r.LogicalName(), err) 865 } 866 defer f.Close() 867 868 // Crawl in data tree to insert data 869 current = s.Data 870 keyParts := strings.Split(r.Dir(), helpers.FilePathSeparator) 871 // The first path element is the virtual folder (typically theme name), which is 872 // not part of the key. 873 if len(keyParts) > 1 { 874 for _, key := range keyParts[1:] { 875 if key != "" { 876 if _, ok := current[key]; !ok { 877 current[key] = make(map[string]interface{}) 878 } 879 current = current[key].(map[string]interface{}) 880 } 881 } 882 } 883 884 data, err := s.readData(r) 885 if err != nil { 886 s.Log.ERROR.Printf("Failed to read data from %s: %s", filepath.Join(r.Path(), r.LogicalName()), err) 887 return nil 888 } 889 890 if data == nil { 891 return nil 892 } 893 894 // filepath.Walk walks the files in lexical order, '/' comes before '.' 895 // this warning could happen if 896 // 1. A theme uses the same key; the main data folder wins 897 // 2. A sub folder uses the same key: the sub folder wins 898 higherPrecedentData := current[r.BaseFileName()] 899 900 switch data.(type) { 901 case nil: 902 // hear the crickets? 903 904 case map[string]interface{}: 905 906 switch higherPrecedentData.(type) { 907 case nil: 908 current[r.BaseFileName()] = data 909 case map[string]interface{}: 910 // merge maps: insert entries from data for keys that 911 // don't already exist in higherPrecedentData 912 higherPrecedentMap := higherPrecedentData.(map[string]interface{}) 913 for key, value := range data.(map[string]interface{}) { 914 if _, exists := higherPrecedentMap[key]; exists { 915 s.Log.WARN.Printf("Data for key '%s' in path '%s' is overridden higher precedence data already in the data tree", key, r.Path()) 916 } else { 917 higherPrecedentMap[key] = value 918 } 919 } 920 default: 921 // can't merge: higherPrecedentData is not a map 922 s.Log.WARN.Printf("The %T data from '%s' overridden by "+ 923 "higher precedence %T data already in the data tree", data, r.Path(), higherPrecedentData) 924 } 925 926 case []interface{}: 927 if higherPrecedentData == nil { 928 current[r.BaseFileName()] = data 929 } else { 930 // we don't merge array data 931 s.Log.WARN.Printf("The %T data from '%s' overridden by "+ 932 "higher precedence %T data already in the data tree", data, r.Path(), higherPrecedentData) 933 } 934 935 default: 936 s.Log.ERROR.Printf("unexpected data type %T in file %s", data, r.LogicalName()) 937 } 938 939 return nil 940 } 941 942 func (s *Site) readData(f source.ReadableFile) (interface{}, error) { 943 file, err := f.Open() 944 if err != nil { 945 return nil, fmt.Errorf("readData: failed to open data file: %s", err) 946 } 947 defer file.Close() 948 content := helpers.ReaderToBytes(file) 949 950 switch f.Extension() { 951 case "yaml", "yml": 952 return parser.HandleYAMLData(content) 953 case "json": 954 return parser.HandleJSONData(content) 955 case "toml": 956 return parser.HandleTOMLMetaData(content) 957 default: 958 return nil, fmt.Errorf("Data not supported for extension '%s'", f.Extension()) 959 } 960 } 961 962 func (s *Site) readDataFromSourceFS() error { 963 err := s.loadData(s.PathSpec.BaseFs.Data.Fs) 964 s.timerStep("load data") 965 return err 966 } 967 968 func (s *Site) process(config BuildCfg) (err error) { 969 if err = s.initialize(); err != nil { 970 return 971 } 972 s.timerStep("initialize") 973 974 if err = s.readDataFromSourceFS(); err != nil { 975 return 976 } 977 978 s.timerStep("load i18n") 979 980 if err := s.readAndProcessContent(); err != nil { 981 return err 982 } 983 s.timerStep("read and convert pages from source") 984 985 return err 986 987 } 988 989 func (s *Site) setupSitePages() { 990 var siteLastChange time.Time 991 992 for i, page := range s.RegularPages { 993 if i < len(s.RegularPages)-1 { 994 page.Next = s.RegularPages[i+1] 995 } 996 997 if i > 0 { 998 page.Prev = s.RegularPages[i-1] 999 } 1000 1001 // Determine Site.Info.LastChange 1002 // Note that the logic to determine which date to use for Lastmod 1003 // is already applied, so this is *the* date to use. 1004 // We cannot just pick the last page in the default sort, because 1005 // that may not be ordered by date. 1006 if page.Lastmod.After(siteLastChange) { 1007 siteLastChange = page.Lastmod 1008 } 1009 } 1010 1011 s.Info.LastChange = siteLastChange 1012 } 1013 1014 func (s *Site) render(config *BuildCfg, outFormatIdx int) (err error) { 1015 1016 if outFormatIdx == 0 { 1017 if err = s.preparePages(); err != nil { 1018 return 1019 } 1020 s.timerStep("prepare pages") 1021 1022 // Note that even if disableAliases is set, the aliases themselves are 1023 // preserved on page. The motivation with this is to be able to generate 1024 // 301 redirects in a .htacess file and similar using a custom output format. 1025 if !s.Cfg.GetBool("disableAliases") { 1026 // Aliases must be rendered before pages. 1027 // Some sites, Hugo docs included, have faulty alias definitions that point 1028 // to itself or another real page. These will be overwritten in the next 1029 // step. 1030 if err = s.renderAliases(); err != nil { 1031 return 1032 } 1033 s.timerStep("render and write aliases") 1034 } 1035 1036 } 1037 1038 if err = s.renderPages(config); err != nil { 1039 return 1040 } 1041 1042 s.timerStep("render and write pages") 1043 1044 // TODO(bep) render consider this, ref. render404 etc. 1045 if outFormatIdx > 0 { 1046 return 1047 } 1048 1049 if err = s.renderSitemap(); err != nil { 1050 return 1051 } 1052 s.timerStep("render and write Sitemap") 1053 1054 if err = s.renderRobotsTXT(); err != nil { 1055 return 1056 } 1057 s.timerStep("render and write robots.txt") 1058 1059 if err = s.render404(); err != nil { 1060 return 1061 } 1062 s.timerStep("render and write 404") 1063 1064 return 1065 } 1066 1067 func (s *Site) Initialise() (err error) { 1068 return s.initialize() 1069 } 1070 1071 func (s *Site) initialize() (err error) { 1072 s.Menus = Menus{} 1073 1074 return s.initializeSiteInfo() 1075 } 1076 1077 // HomeAbsURL is a convenience method giving the absolute URL to the home page. 1078 func (s *SiteInfo) HomeAbsURL() string { 1079 base := "" 1080 if s.IsMultiLingual() { 1081 base = s.Language.Lang 1082 } 1083 return s.owner.AbsURL(base, false) 1084 } 1085 1086 // SitemapAbsURL is a convenience method giving the absolute URL to the sitemap. 1087 func (s *SiteInfo) SitemapAbsURL() string { 1088 sitemapDefault := parseSitemap(s.s.Cfg.GetStringMap("sitemap")) 1089 p := s.HomeAbsURL() 1090 if !strings.HasSuffix(p, "/") { 1091 p += "/" 1092 } 1093 p += sitemapDefault.Filename 1094 return p 1095 } 1096 1097 func (s *Site) initializeSiteInfo() error { 1098 var ( 1099 lang = s.Language 1100 languages langs.Languages 1101 ) 1102 1103 if s.owner != nil && s.owner.multilingual != nil { 1104 languages = s.owner.multilingual.Languages 1105 } 1106 1107 params := lang.Params() 1108 1109 permalinks := make(PermalinkOverrides) 1110 for k, v := range s.Cfg.GetStringMapString("permalinks") { 1111 permalinks[k] = pathPattern(v) 1112 } 1113 1114 defaultContentInSubDir := s.Cfg.GetBool("defaultContentLanguageInSubdir") 1115 defaultContentLanguage := s.Cfg.GetString("defaultContentLanguage") 1116 1117 languagePrefix := "" 1118 if s.multilingualEnabled() && (defaultContentInSubDir || lang.Lang != defaultContentLanguage) { 1119 languagePrefix = "/" + lang.Lang 1120 } 1121 1122 var uglyURLs = func(p *Page) bool { 1123 return false 1124 } 1125 1126 v := s.Cfg.Get("uglyURLs") 1127 if v != nil { 1128 switch vv := v.(type) { 1129 case bool: 1130 uglyURLs = func(p *Page) bool { 1131 return vv 1132 } 1133 case string: 1134 // Is what be get from CLI (--uglyURLs) 1135 vvv := cast.ToBool(vv) 1136 uglyURLs = func(p *Page) bool { 1137 return vvv 1138 } 1139 default: 1140 m := cast.ToStringMapBool(v) 1141 uglyURLs = func(p *Page) bool { 1142 return m[p.Section()] 1143 } 1144 } 1145 } 1146 1147 s.Info = SiteInfo{ 1148 Title: lang.GetString("title"), 1149 Author: lang.GetStringMap("author"), 1150 Social: lang.GetStringMapString("social"), 1151 LanguageCode: lang.GetString("languageCode"), 1152 Copyright: lang.GetString("copyright"), 1153 Language: lang, 1154 LanguagePrefix: languagePrefix, 1155 Languages: languages, 1156 defaultContentLanguageInSubdir: defaultContentInSubDir, 1157 sectionPagesMenu: lang.GetString("sectionPagesMenu"), 1158 BuildDrafts: s.Cfg.GetBool("buildDrafts"), 1159 canonifyURLs: s.Cfg.GetBool("canonifyURLs"), 1160 relativeURLs: s.Cfg.GetBool("relativeURLs"), 1161 uglyURLs: uglyURLs, 1162 preserveTaxonomyNames: lang.GetBool("preserveTaxonomyNames"), 1163 PageCollections: s.PageCollections, 1164 Menus: &s.Menus, 1165 Params: params, 1166 Permalinks: permalinks, 1167 Data: &s.Data, 1168 owner: s.owner, 1169 s: s, 1170 // TODO(bep) make this Menu and similar into delegate methods on SiteInfo 1171 Taxonomies: s.Taxonomies, 1172 } 1173 1174 rssOutputFormat, found := s.outputFormats[KindHome].GetByName(output.RSSFormat.Name) 1175 1176 if found { 1177 s.Info.RSSLink = s.permalink(rssOutputFormat.BaseFilename()) 1178 } 1179 1180 return nil 1181 } 1182 1183 func (s *Site) isI18nEvent(e fsnotify.Event) bool { 1184 return s.BaseFs.SourceFilesystems.IsI18n(e.Name) 1185 } 1186 1187 func (s *Site) isDataDirEvent(e fsnotify.Event) bool { 1188 return s.BaseFs.SourceFilesystems.IsData(e.Name) 1189 } 1190 1191 func (s *Site) isLayoutDirEvent(e fsnotify.Event) bool { 1192 return s.BaseFs.SourceFilesystems.IsLayout(e.Name) 1193 } 1194 1195 func (s *Site) absContentDir() string { 1196 return s.PathSpec.AbsPathify(s.PathSpec.ContentDir) 1197 } 1198 1199 func (s *Site) isContentDirEvent(e fsnotify.Event) bool { 1200 return s.BaseFs.IsContent(e.Name) 1201 } 1202 1203 type contentCaptureResultHandler struct { 1204 defaultContentProcessor *siteContentProcessor 1205 contentProcessors map[string]*siteContentProcessor 1206 } 1207 1208 func (c *contentCaptureResultHandler) getContentProcessor(lang string) *siteContentProcessor { 1209 proc, found := c.contentProcessors[lang] 1210 if found { 1211 return proc 1212 } 1213 return c.defaultContentProcessor 1214 } 1215 1216 func (c *contentCaptureResultHandler) handleSingles(fis ...*fileInfo) { 1217 for _, fi := range fis { 1218 proc := c.getContentProcessor(fi.Lang()) 1219 proc.processSingle(fi) 1220 } 1221 } 1222 func (c *contentCaptureResultHandler) handleBundles(d *bundleDirs) { 1223 for _, b := range d.bundles { 1224 proc := c.getContentProcessor(b.fi.Lang()) 1225 proc.processBundle(b) 1226 } 1227 } 1228 1229 func (c *contentCaptureResultHandler) handleCopyFiles(files ...pathLangFile) { 1230 for _, proc := range c.contentProcessors { 1231 proc.processAssets(files) 1232 } 1233 } 1234 1235 func (s *Site) readAndProcessContent(filenames ...string) error { 1236 ctx := context.Background() 1237 g, ctx := errgroup.WithContext(ctx) 1238 1239 defaultContentLanguage := s.SourceSpec.DefaultContentLanguage 1240 1241 contentProcessors := make(map[string]*siteContentProcessor) 1242 var defaultContentProcessor *siteContentProcessor 1243 sites := s.owner.langSite() 1244 for k, v := range sites { 1245 if v.Language.Disabled { 1246 continue 1247 } 1248 proc := newSiteContentProcessor(ctx, len(filenames) > 0, v) 1249 contentProcessors[k] = proc 1250 if k == defaultContentLanguage { 1251 defaultContentProcessor = proc 1252 } 1253 g.Go(func() error { 1254 return proc.process(ctx) 1255 }) 1256 } 1257 1258 var ( 1259 handler captureResultHandler 1260 bundleMap *contentChangeMap 1261 ) 1262 1263 mainHandler := &contentCaptureResultHandler{contentProcessors: contentProcessors, defaultContentProcessor: defaultContentProcessor} 1264 1265 sourceSpec := source.NewSourceSpec(s.PathSpec, s.BaseFs.Content.Fs) 1266 1267 if s.running() { 1268 // Need to track changes. 1269 bundleMap = s.owner.ContentChanges 1270 handler = &captureResultHandlerChain{handlers: []captureBundlesHandler{mainHandler, bundleMap}} 1271 1272 } else { 1273 handler = mainHandler 1274 } 1275 1276 c := newCapturer(s.Log, sourceSpec, handler, bundleMap, filenames...) 1277 1278 err1 := c.capture() 1279 1280 for _, proc := range contentProcessors { 1281 proc.closeInput() 1282 } 1283 1284 err2 := g.Wait() 1285 1286 if err1 != nil { 1287 return err1 1288 } 1289 return err2 1290 } 1291 1292 func (s *Site) buildSiteMeta() (err error) { 1293 defer s.timerStep("build Site meta") 1294 1295 if len(s.Pages) == 0 { 1296 return 1297 } 1298 1299 s.assembleTaxonomies() 1300 1301 for _, p := range s.AllPages { 1302 // this depends on taxonomies 1303 p.setValuesForKind(s) 1304 } 1305 1306 return 1307 } 1308 1309 func (s *Site) getMenusFromConfig() Menus { 1310 1311 ret := Menus{} 1312 1313 if menus := s.Language.GetStringMap("menu"); menus != nil { 1314 for name, menu := range menus { 1315 m, err := cast.ToSliceE(menu) 1316 if err != nil { 1317 s.Log.ERROR.Printf("unable to process menus in site config\n") 1318 s.Log.ERROR.Println(err) 1319 } else { 1320 for _, entry := range m { 1321 s.Log.DEBUG.Printf("found menu: %q, in site config\n", name) 1322 1323 menuEntry := MenuEntry{Menu: name} 1324 ime, err := cast.ToStringMapE(entry) 1325 if err != nil { 1326 s.Log.ERROR.Printf("unable to process menus in site config\n") 1327 s.Log.ERROR.Println(err) 1328 } 1329 1330 menuEntry.marshallMap(ime) 1331 menuEntry.URL = s.Info.createNodeMenuEntryURL(menuEntry.URL) 1332 1333 if ret[name] == nil { 1334 ret[name] = &Menu{} 1335 } 1336 *ret[name] = ret[name].add(&menuEntry) 1337 } 1338 } 1339 } 1340 return ret 1341 } 1342 return ret 1343 } 1344 1345 func (s *SiteInfo) createNodeMenuEntryURL(in string) string { 1346 1347 if !strings.HasPrefix(in, "/") { 1348 return in 1349 } 1350 // make it match the nodes 1351 menuEntryURL := in 1352 menuEntryURL = helpers.SanitizeURLKeepTrailingSlash(s.s.PathSpec.URLize(menuEntryURL)) 1353 if !s.canonifyURLs { 1354 menuEntryURL = helpers.AddContextRoot(s.s.PathSpec.BaseURL.String(), menuEntryURL) 1355 } 1356 return menuEntryURL 1357 } 1358 1359 func (s *Site) assembleMenus() { 1360 s.Menus = Menus{} 1361 1362 type twoD struct { 1363 MenuName, EntryName string 1364 } 1365 flat := map[twoD]*MenuEntry{} 1366 children := map[twoD]Menu{} 1367 1368 // add menu entries from config to flat hash 1369 menuConfig := s.getMenusFromConfig() 1370 for name, menu := range menuConfig { 1371 for _, me := range *menu { 1372 flat[twoD{name, me.KeyName()}] = me 1373 } 1374 } 1375 1376 sectionPagesMenu := s.Info.sectionPagesMenu 1377 pages := s.Pages 1378 1379 if sectionPagesMenu != "" { 1380 for _, p := range pages { 1381 if p.Kind == KindSection { 1382 // From Hugo 0.22 we have nested sections, but until we get a 1383 // feel of how that would work in this setting, let us keep 1384 // this menu for the top level only. 1385 id := p.Section() 1386 if _, ok := flat[twoD{sectionPagesMenu, id}]; ok { 1387 continue 1388 } 1389 1390 me := MenuEntry{Identifier: id, 1391 Name: p.LinkTitle(), 1392 Weight: p.Weight, 1393 URL: p.RelPermalink()} 1394 flat[twoD{sectionPagesMenu, me.KeyName()}] = &me 1395 } 1396 } 1397 } 1398 1399 // Add menu entries provided by pages 1400 for _, p := range pages { 1401 for name, me := range p.Menus() { 1402 if _, ok := flat[twoD{name, me.KeyName()}]; ok { 1403 s.Log.ERROR.Printf("Two or more menu items have the same name/identifier in Menu %q: %q.\nRename or set an unique identifier.\n", name, me.KeyName()) 1404 continue 1405 } 1406 flat[twoD{name, me.KeyName()}] = me 1407 } 1408 } 1409 1410 // Create Children Menus First 1411 for _, e := range flat { 1412 if e.Parent != "" { 1413 children[twoD{e.Menu, e.Parent}] = children[twoD{e.Menu, e.Parent}].add(e) 1414 } 1415 } 1416 1417 // Placing Children in Parents (in flat) 1418 for p, childmenu := range children { 1419 _, ok := flat[twoD{p.MenuName, p.EntryName}] 1420 if !ok { 1421 // if parent does not exist, create one without a URL 1422 flat[twoD{p.MenuName, p.EntryName}] = &MenuEntry{Name: p.EntryName, URL: ""} 1423 } 1424 flat[twoD{p.MenuName, p.EntryName}].Children = childmenu 1425 } 1426 1427 // Assembling Top Level of Tree 1428 for menu, e := range flat { 1429 if e.Parent == "" { 1430 _, ok := s.Menus[menu.MenuName] 1431 if !ok { 1432 s.Menus[menu.MenuName] = &Menu{} 1433 } 1434 *s.Menus[menu.MenuName] = s.Menus[menu.MenuName].add(e) 1435 } 1436 } 1437 } 1438 1439 func (s *Site) getTaxonomyKey(key string) string { 1440 if s.Info.preserveTaxonomyNames { 1441 // Keep as is 1442 return key 1443 } 1444 return s.PathSpec.MakePathSanitized(key) 1445 } 1446 1447 // We need to create the top level taxonomy early in the build process 1448 // to be able to determine the page Kind correctly. 1449 func (s *Site) createTaxonomiesEntries() { 1450 s.Taxonomies = make(TaxonomyList) 1451 taxonomies := s.Language.GetStringMapString("taxonomies") 1452 for _, plural := range taxonomies { 1453 s.Taxonomies[plural] = make(Taxonomy) 1454 } 1455 } 1456 1457 func (s *Site) assembleTaxonomies() { 1458 s.taxonomiesPluralSingular = make(map[string]string) 1459 s.taxonomiesOrigKey = make(map[string]string) 1460 1461 taxonomies := s.Language.GetStringMapString("taxonomies") 1462 1463 s.Log.INFO.Printf("found taxonomies: %#v\n", taxonomies) 1464 1465 for singular, plural := range taxonomies { 1466 s.taxonomiesPluralSingular[plural] = singular 1467 1468 for _, p := range s.Pages { 1469 vals := p.getParam(plural, !s.Info.preserveTaxonomyNames) 1470 weight := p.getParamToLower(plural + "_weight") 1471 if weight == nil { 1472 weight = 0 1473 } 1474 if vals != nil { 1475 if v, ok := vals.([]string); ok { 1476 for _, idx := range v { 1477 x := WeightedPage{weight.(int), p} 1478 s.Taxonomies[plural].add(s.getTaxonomyKey(idx), x) 1479 if s.Info.preserveTaxonomyNames { 1480 // Need to track the original 1481 s.taxonomiesOrigKey[fmt.Sprintf("%s-%s", plural, s.PathSpec.MakePathSanitized(idx))] = idx 1482 } 1483 } 1484 } else if v, ok := vals.(string); ok { 1485 x := WeightedPage{weight.(int), p} 1486 s.Taxonomies[plural].add(s.getTaxonomyKey(v), x) 1487 if s.Info.preserveTaxonomyNames { 1488 // Need to track the original 1489 s.taxonomiesOrigKey[fmt.Sprintf("%s-%s", plural, s.PathSpec.MakePathSanitized(v))] = v 1490 } 1491 } else { 1492 s.Log.ERROR.Printf("Invalid %s in %s\n", plural, p.File.Path()) 1493 } 1494 } 1495 } 1496 for k := range s.Taxonomies[plural] { 1497 s.Taxonomies[plural][k].Sort() 1498 } 1499 } 1500 1501 s.Info.Taxonomies = s.Taxonomies 1502 } 1503 1504 // Prepare site for a new full build. 1505 func (s *Site) resetBuildState() { 1506 1507 s.relatedDocsHandler = newSearchIndexHandler(s.relatedDocsHandler.cfg) 1508 s.PageCollections = newPageCollectionsFromPages(s.rawAllPages) 1509 // TODO(bep) get rid of this double 1510 s.Info.PageCollections = s.PageCollections 1511 1512 s.draftCount = 0 1513 s.futureCount = 0 1514 1515 s.expiredCount = 0 1516 1517 spc = newPageCache() 1518 1519 for _, p := range s.rawAllPages { 1520 p.subSections = Pages{} 1521 p.parent = nil 1522 p.scratch = maps.NewScratch() 1523 p.mainPageOutput = nil 1524 } 1525 } 1526 1527 func (s *Site) kindFromSections(sections []string) string { 1528 if len(sections) == 0 { 1529 return KindSection 1530 } 1531 1532 if _, isTaxonomy := s.Taxonomies[sections[0]]; isTaxonomy { 1533 if len(sections) == 1 { 1534 return KindTaxonomyTerm 1535 } 1536 return KindTaxonomy 1537 } 1538 return KindSection 1539 } 1540 1541 func (s *Site) layouts(p *PageOutput) ([]string, error) { 1542 return s.layoutHandler.For(p.layoutDescriptor, p.outputFormat) 1543 } 1544 1545 func (s *Site) preparePages() error { 1546 var errors []error 1547 1548 for _, p := range s.Pages { 1549 if err := p.prepareLayouts(); err != nil { 1550 errors = append(errors, err) 1551 } 1552 if err := p.prepareData(s); err != nil { 1553 errors = append(errors, err) 1554 } 1555 } 1556 1557 if len(errors) != 0 { 1558 return fmt.Errorf("Prepare pages failed: %.100q…", errors) 1559 } 1560 1561 return nil 1562 } 1563 1564 func errorCollator(results <-chan error, errs chan<- error) { 1565 errMsgs := []string{} 1566 for err := range results { 1567 if err != nil { 1568 errMsgs = append(errMsgs, err.Error()) 1569 } 1570 } 1571 if len(errMsgs) == 0 { 1572 errs <- nil 1573 } else { 1574 errs <- errors.New(strings.Join(errMsgs, "\n")) 1575 } 1576 close(errs) 1577 } 1578 1579 func (s *Site) appendThemeTemplates(in []string) []string { 1580 if !s.PathSpec.ThemeSet() { 1581 return in 1582 } 1583 1584 out := []string{} 1585 // First place all non internal templates 1586 for _, t := range in { 1587 if !strings.HasPrefix(t, "_internal/") { 1588 out = append(out, t) 1589 } 1590 } 1591 1592 // Then place theme templates with the same names 1593 for _, t := range in { 1594 if !strings.HasPrefix(t, "_internal/") { 1595 out = append(out, "theme/"+t) 1596 } 1597 } 1598 1599 // Lastly place internal templates 1600 for _, t := range in { 1601 if strings.HasPrefix(t, "_internal/") { 1602 out = append(out, t) 1603 } 1604 } 1605 return out 1606 1607 } 1608 1609 // GetPage looks up a page of a given type for the given ref. 1610 // In Hugo <= 0.44 you had to add Page Kind (section, home) etc. as the first 1611 // argument and then either a unix styled path (with or without a leading slash)) 1612 // or path elements separated. 1613 // When we now remove the Kind from this API, we need to make the transition as painless 1614 // as possible for existing sites. Most sites will use {{ .Site.GetPage "section" "my/section" }}, 1615 // i.e. 2 arguments, so we test for that. 1616 func (s *SiteInfo) GetPage(ref ...string) (*Page, error) { 1617 return s.getPageOldVersion(ref...) 1618 } 1619 1620 func (s *Site) permalinkForOutputFormat(link string, f output.Format) (string, error) { 1621 var ( 1622 baseURL string 1623 err error 1624 ) 1625 1626 if f.Protocol != "" { 1627 baseURL, err = s.PathSpec.BaseURL.WithProtocol(f.Protocol) 1628 if err != nil { 1629 return "", err 1630 } 1631 } else { 1632 baseURL = s.PathSpec.BaseURL.String() 1633 } 1634 return s.PathSpec.PermalinkForBaseURL(link, baseURL), nil 1635 } 1636 1637 func (s *Site) permalink(link string) string { 1638 return s.PathSpec.PermalinkForBaseURL(link, s.PathSpec.BaseURL.String()) 1639 1640 } 1641 1642 func (s *Site) renderAndWriteXML(statCounter *uint64, name string, targetPath string, d interface{}, layouts ...string) error { 1643 s.Log.DEBUG.Printf("Render XML for %q to %q", name, targetPath) 1644 renderBuffer := bp.GetBuffer() 1645 defer bp.PutBuffer(renderBuffer) 1646 renderBuffer.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?>\n") 1647 1648 if err := s.renderForLayouts(name, d, renderBuffer, layouts...); err != nil { 1649 helpers.DistinctWarnLog.Println(err) 1650 return nil 1651 } 1652 1653 var path string 1654 if s.Info.relativeURLs { 1655 path = helpers.GetDottedRelativePath(targetPath) 1656 } else { 1657 s := s.PathSpec.BaseURL.String() 1658 if !strings.HasSuffix(s, "/") { 1659 s += "/" 1660 } 1661 path = s 1662 } 1663 1664 pd := publisher.Descriptor{ 1665 Src: renderBuffer, 1666 TargetPath: targetPath, 1667 StatCounter: statCounter, 1668 // For the minification part of XML, 1669 // we currently only use the MIME type. 1670 OutputFormat: output.RSSFormat, 1671 AbsURLPath: path, 1672 } 1673 1674 return s.publisher.Publish(pd) 1675 1676 } 1677 1678 func (s *Site) renderAndWritePage(statCounter *uint64, name string, targetPath string, p *PageOutput, layouts ...string) error { 1679 renderBuffer := bp.GetBuffer() 1680 defer bp.PutBuffer(renderBuffer) 1681 1682 if err := s.renderForLayouts(p.Kind, p, renderBuffer, layouts...); err != nil { 1683 helpers.DistinctWarnLog.Println(err) 1684 return nil 1685 } 1686 1687 if renderBuffer.Len() == 0 { 1688 return nil 1689 } 1690 1691 isHTML := p.outputFormat.IsHTML 1692 1693 var path string 1694 1695 if s.Info.relativeURLs { 1696 path = helpers.GetDottedRelativePath(targetPath) 1697 } else if s.Info.canonifyURLs { 1698 url := s.PathSpec.BaseURL.String() 1699 if !strings.HasSuffix(url, "/") { 1700 url += "/" 1701 } 1702 path = url 1703 } 1704 1705 pd := publisher.Descriptor{ 1706 Src: renderBuffer, 1707 TargetPath: targetPath, 1708 StatCounter: statCounter, 1709 OutputFormat: p.outputFormat, 1710 } 1711 1712 if isHTML { 1713 if s.Info.relativeURLs || s.Info.canonifyURLs { 1714 pd.AbsURLPath = path 1715 } 1716 1717 if s.running() && s.Cfg.GetBool("watch") && !s.Cfg.GetBool("disableLiveReload") { 1718 pd.LiveReloadPort = s.Cfg.GetInt("liveReloadPort") 1719 } 1720 1721 // For performance reasons we only inject the Hugo generator tag on the home page. 1722 if p.IsHome() { 1723 pd.AddHugoGeneratorTag = !s.Cfg.GetBool("disableHugoGeneratorInject") 1724 } 1725 1726 } 1727 1728 return s.publisher.Publish(pd) 1729 } 1730 1731 func (s *Site) renderForLayouts(name string, d interface{}, w io.Writer, layouts ...string) (err error) { 1732 var templ tpl.Template 1733 1734 defer func() { 1735 if r := recover(); r != nil { 1736 templName := "" 1737 if templ != nil { 1738 templName = templ.Name() 1739 } 1740 s.DistinctErrorLog.Printf("Failed to render %q: %s", templName, r) 1741 s.DistinctErrorLog.Printf("Stack Trace:\n%s", stackTrace(1200)) 1742 1743 // TOD(bep) we really need to fix this. Also see below. 1744 if !s.running() && !testMode { 1745 os.Exit(-1) 1746 } 1747 } 1748 }() 1749 1750 templ = s.findFirstTemplate(layouts...) 1751 if templ == nil { 1752 return fmt.Errorf("[%s] Unable to locate layout for %q: %s\n", s.Language.Lang, name, layouts) 1753 } 1754 1755 if err = templ.Execute(w, d); err != nil { 1756 // Behavior here should be dependent on if running in server or watch mode. 1757 if p, ok := d.(*PageOutput); ok { 1758 if p.File != nil { 1759 s.DistinctErrorLog.Printf("Error while rendering %q in %q: %s", name, p.File.Dir(), err) 1760 } else { 1761 s.DistinctErrorLog.Printf("Error while rendering %q: %s", name, err) 1762 } 1763 } else { 1764 s.DistinctErrorLog.Printf("Error while rendering %q: %s", name, err) 1765 } 1766 if !s.running() && !testMode { 1767 // TODO(bep) check if this can be propagated 1768 os.Exit(-1) 1769 } else if testMode { 1770 return 1771 } 1772 } 1773 1774 return 1775 } 1776 1777 func (s *Site) findFirstTemplate(layouts ...string) tpl.Template { 1778 for _, layout := range layouts { 1779 if templ, found := s.Tmpl.Lookup(layout); found { 1780 return templ 1781 } 1782 } 1783 return nil 1784 } 1785 1786 func (s *Site) publish(statCounter *uint64, path string, r io.Reader) (err error) { 1787 s.PathSpec.ProcessingStats.Incr(statCounter) 1788 1789 return helpers.WriteToDisk(filepath.Clean(path), r, s.BaseFs.PublishFs) 1790 } 1791 1792 func getGoMaxProcs() int { 1793 if gmp := os.Getenv("GOMAXPROCS"); gmp != "" { 1794 if p, err := strconv.Atoi(gmp); err != nil { 1795 return p 1796 } 1797 } 1798 return 1 1799 } 1800 1801 func (s *Site) newNodePage(typ string, sections ...string) *Page { 1802 p := &Page{ 1803 language: s.Language, 1804 pageInit: &pageInit{}, 1805 pageContentInit: &pageContentInit{}, 1806 Kind: typ, 1807 Source: Source{File: &source.FileInfo{}}, 1808 data: make(map[string]interface{}), 1809 Site: &s.Info, 1810 sections: sections, 1811 s: s} 1812 1813 p.outputFormats = p.s.outputFormats[p.Kind] 1814 1815 return p 1816 1817 } 1818 1819 func (s *Site) newHomePage() *Page { 1820 p := s.newNodePage(KindHome) 1821 p.title = s.Info.Title 1822 pages := Pages{} 1823 p.data["Pages"] = pages 1824 p.Pages = pages 1825 return p 1826 } 1827 1828 func (s *Site) newTaxonomyPage(plural, key string) *Page { 1829 1830 p := s.newNodePage(KindTaxonomy, plural, key) 1831 1832 if s.Info.preserveTaxonomyNames { 1833 // Keep (mostly) as is in the title 1834 // We make the first character upper case, mostly because 1835 // it is easier to reason about in the tests. 1836 p.title = helpers.FirstUpper(key) 1837 key = s.PathSpec.MakePathSanitized(key) 1838 } else { 1839 p.title = strings.Replace(s.titleFunc(key), "-", " ", -1) 1840 } 1841 1842 return p 1843 } 1844 1845 func (s *Site) newSectionPage(name string) *Page { 1846 p := s.newNodePage(KindSection, name) 1847 1848 sectionName := helpers.FirstUpper(name) 1849 if s.Cfg.GetBool("pluralizeListTitles") { 1850 p.title = inflect.Pluralize(sectionName) 1851 } else { 1852 p.title = sectionName 1853 } 1854 return p 1855 } 1856 1857 func (s *Site) newTaxonomyTermsPage(plural string) *Page { 1858 p := s.newNodePage(KindTaxonomyTerm, plural) 1859 p.title = s.titleFunc(plural) 1860 return p 1861 }