github.com/shohhei1126/hugo@v0.42.2-0.20180623210752-3d5928889ad7/hugolib/site.go (about) 1 // Copyright 2017 The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package hugolib 15 16 import ( 17 "errors" 18 "fmt" 19 "html/template" 20 "io" 21 "mime" 22 "net/url" 23 "os" 24 "path/filepath" 25 "sort" 26 "strconv" 27 "strings" 28 "time" 29 30 "github.com/gohugoio/hugo/langs" 31 32 src "github.com/gohugoio/hugo/source" 33 34 "github.com/gohugoio/hugo/resource" 35 36 "golang.org/x/sync/errgroup" 37 38 "github.com/gohugoio/hugo/config" 39 40 "github.com/gohugoio/hugo/media" 41 42 "github.com/markbates/inflect" 43 "golang.org/x/net/context" 44 45 "github.com/fsnotify/fsnotify" 46 bp "github.com/gohugoio/hugo/bufferpool" 47 "github.com/gohugoio/hugo/deps" 48 "github.com/gohugoio/hugo/helpers" 49 "github.com/gohugoio/hugo/hugolib/pagemeta" 50 "github.com/gohugoio/hugo/output" 51 "github.com/gohugoio/hugo/parser" 52 "github.com/gohugoio/hugo/related" 53 "github.com/gohugoio/hugo/source" 54 "github.com/gohugoio/hugo/tpl" 55 "github.com/gohugoio/hugo/transform" 56 "github.com/spf13/afero" 57 "github.com/spf13/cast" 58 "github.com/spf13/nitro" 59 "github.com/spf13/viper" 60 ) 61 62 var _ = transform.AbsURL 63 64 // used to indicate if run as a test. 65 var testMode bool 66 67 var defaultTimer *nitro.B 68 69 // Site contains all the information relevant for constructing a static 70 // site. The basic flow of information is as follows: 71 // 72 // 1. A list of Files is parsed and then converted into Pages. 73 // 74 // 2. Pages contain sections (based on the file they were generated from), 75 // aliases and slugs (included in a pages frontmatter) which are the 76 // various targets that will get generated. There will be canonical 77 // listing. The canonical path can be overruled based on a pattern. 78 // 79 // 3. Taxonomies are created via configuration and will present some aspect of 80 // the final page and typically a perm url. 81 // 82 // 4. All Pages are passed through a template based on their desired 83 // layout based on numerous different elements. 84 // 85 // 5. The entire collection of files is written to disk. 86 type Site struct { 87 owner *HugoSites 88 89 *PageCollections 90 91 Taxonomies TaxonomyList 92 93 // Plural is what we get in the folder, so keep track of this mapping 94 // to get the singular form from that value. 95 taxonomiesPluralSingular map[string]string 96 97 // This is temporary, see https://github.com/gohugoio/hugo/issues/2835 98 // Maps "actors-gerard-depardieu" to "Gérard Depardieu" when preserveTaxonomyNames 99 // is set. 100 taxonomiesOrigKey map[string]string 101 102 Sections Taxonomy 103 Info SiteInfo 104 Menus Menus 105 timer *nitro.B 106 107 layoutHandler *output.LayoutHandler 108 109 draftCount int 110 futureCount int 111 expiredCount int 112 113 Data map[string]interface{} 114 Language *langs.Language 115 116 disabledKinds map[string]bool 117 118 // Output formats defined in site config per Page Kind, or some defaults 119 // if not set. 120 // Output formats defined in Page front matter will override these. 121 outputFormats map[string]output.Formats 122 123 // All the output formats and media types available for this site. 124 // These values will be merged from the Hugo defaults, the site config and, 125 // finally, the language settings. 126 outputFormatsConfig output.Formats 127 mediaTypesConfig media.Types 128 129 // How to handle page front matter. 130 frontmatterHandler pagemeta.FrontMatterHandler 131 132 // We render each site for all the relevant output formats in serial with 133 // this rendering context pointing to the current one. 134 rc *siteRenderingContext 135 136 // The output formats that we need to render this site in. This slice 137 // will be fixed once set. 138 // This will be the union of Site.Pages' outputFormats. 139 // This slice will be sorted. 140 renderFormats output.Formats 141 142 // Logger etc. 143 *deps.Deps `json:"-"` 144 resourceSpec *resource.Spec 145 146 // The func used to title case titles. 147 titleFunc func(s string) string 148 149 relatedDocsHandler *relatedDocsHandler 150 } 151 152 type siteRenderingContext struct { 153 output.Format 154 } 155 156 func (s *Site) initRenderFormats() { 157 formatSet := make(map[string]bool) 158 formats := output.Formats{} 159 for _, p := range s.Pages { 160 for _, f := range p.outputFormats { 161 if !formatSet[f.Name] { 162 formats = append(formats, f) 163 formatSet[f.Name] = true 164 } 165 } 166 } 167 168 sort.Sort(formats) 169 s.renderFormats = formats 170 } 171 172 func (s *Site) isEnabled(kind string) bool { 173 if kind == kindUnknown { 174 panic("Unknown kind") 175 } 176 return !s.disabledKinds[kind] 177 } 178 179 // reset returns a new Site prepared for rebuild. 180 func (s *Site) reset() *Site { 181 return &Site{Deps: s.Deps, 182 layoutHandler: output.NewLayoutHandler(), 183 disabledKinds: s.disabledKinds, 184 titleFunc: s.titleFunc, 185 relatedDocsHandler: newSearchIndexHandler(s.relatedDocsHandler.cfg), 186 outputFormats: s.outputFormats, 187 rc: s.rc, 188 outputFormatsConfig: s.outputFormatsConfig, 189 frontmatterHandler: s.frontmatterHandler, 190 mediaTypesConfig: s.mediaTypesConfig, 191 resourceSpec: s.resourceSpec, 192 Language: s.Language, 193 owner: s.owner, 194 PageCollections: newPageCollections()} 195 } 196 197 // newSite creates a new site with the given configuration. 198 func newSite(cfg deps.DepsCfg) (*Site, error) { 199 c := newPageCollections() 200 201 if cfg.Language == nil { 202 cfg.Language = langs.NewDefaultLanguage(cfg.Cfg) 203 } 204 205 disabledKinds := make(map[string]bool) 206 for _, disabled := range cast.ToStringSlice(cfg.Language.Get("disableKinds")) { 207 disabledKinds[disabled] = true 208 } 209 210 var ( 211 mediaTypesConfig []map[string]interface{} 212 outputFormatsConfig []map[string]interface{} 213 214 siteOutputFormatsConfig output.Formats 215 siteMediaTypesConfig media.Types 216 err error 217 ) 218 219 // Add language last, if set, so it gets precedence. 220 for _, cfg := range []config.Provider{cfg.Cfg, cfg.Language} { 221 if cfg.IsSet("mediaTypes") { 222 mediaTypesConfig = append(mediaTypesConfig, cfg.GetStringMap("mediaTypes")) 223 } 224 if cfg.IsSet("outputFormats") { 225 outputFormatsConfig = append(outputFormatsConfig, cfg.GetStringMap("outputFormats")) 226 } 227 } 228 229 siteMediaTypesConfig, err = media.DecodeTypes(mediaTypesConfig...) 230 if err != nil { 231 return nil, err 232 } 233 234 siteOutputFormatsConfig, err = output.DecodeFormats(siteMediaTypesConfig, outputFormatsConfig...) 235 if err != nil { 236 return nil, err 237 } 238 239 outputFormats, err := createSiteOutputFormats(siteOutputFormatsConfig, cfg.Language) 240 if err != nil { 241 return nil, err 242 } 243 244 var relatedContentConfig related.Config 245 246 if cfg.Language.IsSet("related") { 247 relatedContentConfig, err = related.DecodeConfig(cfg.Language.Get("related")) 248 if err != nil { 249 return nil, err 250 } 251 } else { 252 relatedContentConfig = related.DefaultConfig 253 taxonomies := cfg.Language.GetStringMapString("taxonomies") 254 if _, found := taxonomies["tag"]; found { 255 relatedContentConfig.Add(related.IndexConfig{Name: "tags", Weight: 80}) 256 } 257 } 258 259 titleFunc := helpers.GetTitleFunc(cfg.Language.GetString("titleCaseStyle")) 260 261 frontMatterHandler, err := pagemeta.NewFrontmatterHandler(cfg.Logger, cfg.Cfg) 262 if err != nil { 263 return nil, err 264 } 265 266 s := &Site{ 267 PageCollections: c, 268 layoutHandler: output.NewLayoutHandler(), 269 Language: cfg.Language, 270 disabledKinds: disabledKinds, 271 titleFunc: titleFunc, 272 relatedDocsHandler: newSearchIndexHandler(relatedContentConfig), 273 outputFormats: outputFormats, 274 rc: &siteRenderingContext{output.HTMLFormat}, 275 outputFormatsConfig: siteOutputFormatsConfig, 276 mediaTypesConfig: siteMediaTypesConfig, 277 frontmatterHandler: frontMatterHandler, 278 } 279 280 s.Info = newSiteInfo(siteBuilderCfg{s: s, pageCollections: c, language: s.Language}) 281 282 return s, nil 283 284 } 285 286 // NewSite creates a new site with the given dependency configuration. 287 // The site will have a template system loaded and ready to use. 288 // Note: This is mainly used in single site tests. 289 func NewSite(cfg deps.DepsCfg) (*Site, error) { 290 s, err := newSite(cfg) 291 if err != nil { 292 return nil, err 293 } 294 295 if err = applyDepsIfNeeded(cfg, s); err != nil { 296 return nil, err 297 } 298 299 return s, nil 300 } 301 302 // NewSiteDefaultLang creates a new site in the default language. 303 // The site will have a template system loaded and ready to use. 304 // Note: This is mainly used in single site tests. 305 // TODO(bep) test refactor -- remove 306 func NewSiteDefaultLang(withTemplate ...func(templ tpl.TemplateHandler) error) (*Site, error) { 307 v := viper.New() 308 if err := loadDefaultSettingsFor(v); err != nil { 309 return nil, err 310 } 311 return newSiteForLang(langs.NewDefaultLanguage(v), withTemplate...) 312 } 313 314 // NewEnglishSite creates a new site in English language. 315 // The site will have a template system loaded and ready to use. 316 // Note: This is mainly used in single site tests. 317 // TODO(bep) test refactor -- remove 318 func NewEnglishSite(withTemplate ...func(templ tpl.TemplateHandler) error) (*Site, error) { 319 v := viper.New() 320 if err := loadDefaultSettingsFor(v); err != nil { 321 return nil, err 322 } 323 return newSiteForLang(langs.NewLanguage("en", v), withTemplate...) 324 } 325 326 // newSiteForLang creates a new site in the given language. 327 func newSiteForLang(lang *langs.Language, withTemplate ...func(templ tpl.TemplateHandler) error) (*Site, error) { 328 withTemplates := func(templ tpl.TemplateHandler) error { 329 for _, wt := range withTemplate { 330 if err := wt(templ); err != nil { 331 return err 332 } 333 } 334 return nil 335 } 336 337 cfg := deps.DepsCfg{WithTemplate: withTemplates, Language: lang, Cfg: lang} 338 339 return NewSiteForCfg(cfg) 340 341 } 342 343 // NewSiteForCfg creates a new site for the given configuration. 344 // The site will have a template system loaded and ready to use. 345 // Note: This is mainly used in single site tests. 346 func NewSiteForCfg(cfg deps.DepsCfg) (*Site, error) { 347 s, err := newSite(cfg) 348 349 if err != nil { 350 return nil, err 351 } 352 353 if err := applyDepsIfNeeded(cfg, s); err != nil { 354 return nil, err 355 } 356 return s, nil 357 } 358 359 type SiteInfos []*SiteInfo 360 361 // First is a convenience method to get the first Site, i.e. the main language. 362 func (s SiteInfos) First() *SiteInfo { 363 if len(s) == 0 { 364 return nil 365 } 366 return s[0] 367 } 368 369 type SiteInfo struct { 370 Taxonomies TaxonomyList 371 Authors AuthorList 372 Social SiteSocial 373 *PageCollections 374 Menus *Menus 375 Hugo *HugoInfo 376 Title string 377 RSSLink string 378 Author map[string]interface{} 379 LanguageCode string 380 Copyright string 381 LastChange time.Time 382 Permalinks PermalinkOverrides 383 Params map[string]interface{} 384 BuildDrafts bool 385 canonifyURLs bool 386 relativeURLs bool 387 uglyURLs func(p *Page) bool 388 preserveTaxonomyNames bool 389 Data *map[string]interface{} 390 391 Config SiteConfig 392 393 owner *HugoSites 394 s *Site 395 multilingual *Multilingual 396 Language *langs.Language 397 LanguagePrefix string 398 Languages langs.Languages 399 defaultContentLanguageInSubdir bool 400 sectionPagesMenu string 401 } 402 403 func (s *SiteInfo) String() string { 404 return fmt.Sprintf("Site(%q)", s.Title) 405 } 406 407 func (s *SiteInfo) BaseURL() template.URL { 408 return template.URL(s.s.PathSpec.BaseURL.String()) 409 } 410 411 // ServerPort returns the port part of the BaseURL, 0 if none found. 412 func (s *SiteInfo) ServerPort() int { 413 ps := s.s.PathSpec.BaseURL.URL().Port() 414 if ps == "" { 415 return 0 416 } 417 p, err := strconv.Atoi(ps) 418 if err != nil { 419 return 0 420 } 421 return p 422 } 423 424 // GoogleAnalytics is kept here for historic reasons. 425 func (s *SiteInfo) GoogleAnalytics() string { 426 return s.Config.Services.GoogleAnalytics.ID 427 428 } 429 430 // DisqusShortname is kept here for historic reasons. 431 func (s *SiteInfo) DisqusShortname() string { 432 return s.Config.Services.Disqus.Shortname 433 } 434 435 // Used in tests. 436 437 type siteBuilderCfg struct { 438 language *langs.Language 439 s *Site 440 pageCollections *PageCollections 441 } 442 443 // TODO(bep) get rid of this 444 func newSiteInfo(cfg siteBuilderCfg) SiteInfo { 445 return SiteInfo{ 446 s: cfg.s, 447 multilingual: newMultiLingualForLanguage(cfg.language), 448 PageCollections: cfg.pageCollections, 449 Params: make(map[string]interface{}), 450 uglyURLs: func(p *Page) bool { 451 return false 452 }, 453 } 454 } 455 456 // SiteSocial is a place to put social details on a site level. These are the 457 // standard keys that themes will expect to have available, but can be 458 // expanded to any others on a per site basis 459 // github 460 // facebook 461 // facebook_admin 462 // twitter 463 // twitter_domain 464 // googleplus 465 // pinterest 466 // instagram 467 // youtube 468 // linkedin 469 type SiteSocial map[string]string 470 471 // Param is a convenience method to do lookups in SiteInfo's Params map. 472 // 473 // This method is also implemented on Page and Node. 474 func (s *SiteInfo) Param(key interface{}) (interface{}, error) { 475 keyStr, err := cast.ToStringE(key) 476 if err != nil { 477 return nil, err 478 } 479 keyStr = strings.ToLower(keyStr) 480 return s.Params[keyStr], nil 481 } 482 483 func (s *SiteInfo) IsMultiLingual() bool { 484 return len(s.Languages) > 1 485 } 486 487 func (s *SiteInfo) IsServer() bool { 488 return s.owner.running 489 } 490 491 func (s *SiteInfo) refLink(ref string, page *Page, relative bool, outputFormat string) (string, error) { 492 var refURL *url.URL 493 var err error 494 495 ref = filepath.ToSlash(ref) 496 ref = strings.TrimPrefix(ref, "/") 497 498 refURL, err = url.Parse(ref) 499 500 if err != nil { 501 return "", err 502 } 503 504 var target *Page 505 var link string 506 507 if refURL.Path != "" { 508 target := s.getPage(KindPage, refURL.Path) 509 510 if target == nil { 511 return "", fmt.Errorf("No page found with path or logical name \"%s\".\n", refURL.Path) 512 } 513 514 var permalinker Permalinker = target 515 516 if outputFormat != "" { 517 o := target.OutputFormats().Get(outputFormat) 518 519 if o == nil { 520 return "", fmt.Errorf("Output format %q not found for page %q", outputFormat, refURL.Path) 521 } 522 permalinker = o 523 } 524 525 if relative { 526 link = permalinker.RelPermalink() 527 } else { 528 link = permalinker.Permalink() 529 } 530 } 531 532 if refURL.Fragment != "" { 533 link = link + "#" + refURL.Fragment 534 535 if refURL.Path != "" && target != nil && !target.getRenderingConfig().PlainIDAnchors { 536 link = link + ":" + target.UniqueID() 537 } else if page != nil && !page.getRenderingConfig().PlainIDAnchors { 538 link = link + ":" + page.UniqueID() 539 } 540 } 541 542 return link, nil 543 } 544 545 // Ref will give an absolute URL to ref in the given Page. 546 func (s *SiteInfo) Ref(ref string, page *Page, options ...string) (string, error) { 547 outputFormat := "" 548 if len(options) > 0 { 549 outputFormat = options[0] 550 } 551 552 return s.refLink(ref, page, false, outputFormat) 553 } 554 555 // RelRef will give an relative URL to ref in the given Page. 556 func (s *SiteInfo) RelRef(ref string, page *Page, options ...string) (string, error) { 557 outputFormat := "" 558 if len(options) > 0 { 559 outputFormat = options[0] 560 } 561 562 return s.refLink(ref, page, true, outputFormat) 563 } 564 565 func (s *Site) running() bool { 566 return s.owner != nil && s.owner.running 567 } 568 569 func init() { 570 defaultTimer = nitro.Initalize() 571 } 572 573 func (s *Site) timerStep(step string) { 574 if s.timer == nil { 575 s.timer = defaultTimer 576 } 577 s.timer.Step(step) 578 } 579 580 type whatChanged struct { 581 source bool 582 other bool 583 files map[string]bool 584 } 585 586 // RegisterMediaTypes will register the Site's media types in the mime 587 // package, so it will behave correctly with Hugo's built-in server. 588 func (s *Site) RegisterMediaTypes() { 589 for _, mt := range s.mediaTypesConfig { 590 // The last one will win if there are any duplicates. 591 _ = mime.AddExtensionType("."+mt.Suffix, mt.Type()+"; charset=utf-8") 592 } 593 } 594 595 func (s *Site) filterFileEvents(events []fsnotify.Event) []fsnotify.Event { 596 var filtered []fsnotify.Event 597 seen := make(map[fsnotify.Event]bool) 598 599 for _, ev := range events { 600 // Avoid processing the same event twice. 601 if seen[ev] { 602 continue 603 } 604 seen[ev] = true 605 606 if s.SourceSpec.IgnoreFile(ev.Name) { 607 continue 608 } 609 610 // Throw away any directories 611 isRegular, err := s.SourceSpec.IsRegularSourceFile(ev.Name) 612 if err != nil && os.IsNotExist(err) && (ev.Op&fsnotify.Remove == fsnotify.Remove || ev.Op&fsnotify.Rename == fsnotify.Rename) { 613 // Force keep of event 614 isRegular = true 615 } 616 if !isRegular { 617 continue 618 } 619 620 filtered = append(filtered, ev) 621 } 622 623 return filtered 624 } 625 626 func (s *Site) translateFileEvents(events []fsnotify.Event) []fsnotify.Event { 627 var filtered []fsnotify.Event 628 629 eventMap := make(map[string][]fsnotify.Event) 630 631 // We often get a Remove etc. followed by a Create, a Create followed by a Write. 632 // Remove the superflous events to mage the update logic simpler. 633 for _, ev := range events { 634 eventMap[ev.Name] = append(eventMap[ev.Name], ev) 635 } 636 637 for _, ev := range events { 638 mapped := eventMap[ev.Name] 639 640 // Keep one 641 found := false 642 var kept fsnotify.Event 643 for i, ev2 := range mapped { 644 if i == 0 { 645 kept = ev2 646 } 647 648 if ev2.Op&fsnotify.Write == fsnotify.Write { 649 kept = ev2 650 found = true 651 } 652 653 if !found && ev2.Op&fsnotify.Create == fsnotify.Create { 654 kept = ev2 655 } 656 } 657 658 filtered = append(filtered, kept) 659 } 660 661 return filtered 662 } 663 664 // reBuild partially rebuilds a site given the filesystem events. 665 // It returns whetever the content source was changed. 666 // TODO(bep) clean up/rewrite this method. 667 func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) { 668 669 events = s.filterFileEvents(events) 670 events = s.translateFileEvents(events) 671 672 s.Log.DEBUG.Printf("Rebuild for events %q", events) 673 674 h := s.owner 675 676 s.timerStep("initialize rebuild") 677 678 // First we need to determine what changed 679 680 var ( 681 sourceChanged = []fsnotify.Event{} 682 sourceReallyChanged = []fsnotify.Event{} 683 contentFilesChanged []string 684 tmplChanged = []fsnotify.Event{} 685 dataChanged = []fsnotify.Event{} 686 i18nChanged = []fsnotify.Event{} 687 shortcodesChanged = make(map[string]bool) 688 sourceFilesChanged = make(map[string]bool) 689 690 // prevent spamming the log on changes 691 logger = helpers.NewDistinctFeedbackLogger() 692 ) 693 694 for _, ev := range events { 695 if s.isContentDirEvent(ev) { 696 logger.Println("Source changed", ev) 697 sourceChanged = append(sourceChanged, ev) 698 } 699 if s.isLayoutDirEvent(ev) { 700 logger.Println("Template changed", ev) 701 tmplChanged = append(tmplChanged, ev) 702 703 if strings.Contains(ev.Name, "shortcodes") { 704 clearIsInnerShortcodeCache() 705 shortcode := filepath.Base(ev.Name) 706 shortcode = strings.TrimSuffix(shortcode, filepath.Ext(shortcode)) 707 shortcodesChanged[shortcode] = true 708 } 709 } 710 if s.isDataDirEvent(ev) { 711 logger.Println("Data changed", ev) 712 dataChanged = append(dataChanged, ev) 713 } 714 if s.isI18nEvent(ev) { 715 logger.Println("i18n changed", ev) 716 i18nChanged = append(dataChanged, ev) 717 } 718 } 719 720 if len(tmplChanged) > 0 || len(i18nChanged) > 0 { 721 sites := s.owner.Sites 722 first := sites[0] 723 724 // TOD(bep) globals clean 725 if err := first.Deps.LoadResources(); err != nil { 726 s.Log.ERROR.Println(err) 727 } 728 729 s.TemplateHandler().PrintErrors() 730 731 for i := 1; i < len(sites); i++ { 732 site := sites[i] 733 var err error 734 site.Deps, err = first.Deps.ForLanguage(site.Language) 735 if err != nil { 736 return whatChanged{}, err 737 } 738 } 739 740 s.timerStep("template prep") 741 } 742 743 if len(dataChanged) > 0 { 744 if err := s.readDataFromSourceFS(); err != nil { 745 s.Log.ERROR.Println(err) 746 } 747 } 748 749 for _, ev := range sourceChanged { 750 removed := false 751 752 if ev.Op&fsnotify.Remove == fsnotify.Remove { 753 removed = true 754 } 755 756 // Some editors (Vim) sometimes issue only a Rename operation when writing an existing file 757 // Sometimes a rename operation means that file has been renamed other times it means 758 // it's been updated 759 if ev.Op&fsnotify.Rename == fsnotify.Rename { 760 // If the file is still on disk, it's only been updated, if it's not, it's been moved 761 if ex, err := afero.Exists(s.Fs.Source, ev.Name); !ex || err != nil { 762 removed = true 763 } 764 } 765 if removed && isContentFile(ev.Name) { 766 h.removePageByFilename(ev.Name) 767 } 768 769 sourceReallyChanged = append(sourceReallyChanged, ev) 770 sourceFilesChanged[ev.Name] = true 771 } 772 773 for shortcode := range shortcodesChanged { 774 // There are certain scenarios that, when a shortcode changes, 775 // it isn't sufficient to just rerender the already parsed shortcode. 776 // One example is if the user adds a new shortcode to the content file first, 777 // and then creates the shortcode on the file system. 778 // To handle these scenarios, we must do a full reprocessing of the 779 // pages that keeps a reference to the changed shortcode. 780 pagesWithShortcode := h.findPagesByShortcode(shortcode) 781 for _, p := range pagesWithShortcode { 782 contentFilesChanged = append(contentFilesChanged, p.File.Filename()) 783 } 784 } 785 786 if len(sourceReallyChanged) > 0 || len(contentFilesChanged) > 0 { 787 var filenamesChanged []string 788 for _, e := range sourceReallyChanged { 789 filenamesChanged = append(filenamesChanged, e.Name) 790 } 791 if len(contentFilesChanged) > 0 { 792 filenamesChanged = append(filenamesChanged, contentFilesChanged...) 793 } 794 795 filenamesChanged = helpers.UniqueStrings(filenamesChanged) 796 797 if err := s.readAndProcessContent(filenamesChanged...); err != nil { 798 return whatChanged{}, err 799 } 800 } 801 802 changed := whatChanged{ 803 source: len(sourceChanged) > 0, 804 other: len(tmplChanged) > 0 || len(i18nChanged) > 0 || len(dataChanged) > 0, 805 files: sourceFilesChanged, 806 } 807 808 return changed, nil 809 810 } 811 812 func (s *Site) loadData(fs afero.Fs) (err error) { 813 spec := src.NewSourceSpec(s.PathSpec, fs) 814 fileSystem := spec.NewFilesystem("") 815 s.Data = make(map[string]interface{}) 816 for _, r := range fileSystem.Files() { 817 if err := s.handleDataFile(r); err != nil { 818 return err 819 } 820 } 821 822 return 823 } 824 825 func (s *Site) handleDataFile(r source.ReadableFile) error { 826 var current map[string]interface{} 827 828 f, err := r.Open() 829 if err != nil { 830 return fmt.Errorf("Failed to open data file %q: %s", r.LogicalName(), err) 831 } 832 defer f.Close() 833 834 // Crawl in data tree to insert data 835 current = s.Data 836 keyParts := strings.Split(r.Dir(), helpers.FilePathSeparator) 837 // The first path element is the virtual folder (typically theme name), which is 838 // not part of the key. 839 if len(keyParts) > 1 { 840 for _, key := range keyParts[1:] { 841 if key != "" { 842 if _, ok := current[key]; !ok { 843 current[key] = make(map[string]interface{}) 844 } 845 current = current[key].(map[string]interface{}) 846 } 847 } 848 } 849 850 data, err := s.readData(r) 851 if err != nil { 852 s.Log.ERROR.Printf("Failed to read data from %s: %s", filepath.Join(r.Path(), r.LogicalName()), err) 853 return nil 854 } 855 856 if data == nil { 857 return nil 858 } 859 860 // filepath.Walk walks the files in lexical order, '/' comes before '.' 861 // this warning could happen if 862 // 1. A theme uses the same key; the main data folder wins 863 // 2. A sub folder uses the same key: the sub folder wins 864 higherPrecedentData := current[r.BaseFileName()] 865 866 switch data.(type) { 867 case nil: 868 // hear the crickets? 869 870 case map[string]interface{}: 871 872 switch higherPrecedentData.(type) { 873 case nil: 874 current[r.BaseFileName()] = data 875 case map[string]interface{}: 876 // merge maps: insert entries from data for keys that 877 // don't already exist in higherPrecedentData 878 higherPrecedentMap := higherPrecedentData.(map[string]interface{}) 879 for key, value := range data.(map[string]interface{}) { 880 if _, exists := higherPrecedentMap[key]; exists { 881 s.Log.WARN.Printf("Data for key '%s' in path '%s' is overridden higher precedence data already in the data tree", key, r.Path()) 882 } else { 883 higherPrecedentMap[key] = value 884 } 885 } 886 default: 887 // can't merge: higherPrecedentData is not a map 888 s.Log.WARN.Printf("The %T data from '%s' overridden by "+ 889 "higher precedence %T data already in the data tree", data, r.Path(), higherPrecedentData) 890 } 891 892 case []interface{}: 893 if higherPrecedentData == nil { 894 current[r.BaseFileName()] = data 895 } else { 896 // we don't merge array data 897 s.Log.WARN.Printf("The %T data from '%s' overridden by "+ 898 "higher precedence %T data already in the data tree", data, r.Path(), higherPrecedentData) 899 } 900 901 default: 902 s.Log.ERROR.Printf("unexpected data type %T in file %s", data, r.LogicalName()) 903 } 904 905 return nil 906 } 907 908 func (s *Site) readData(f source.ReadableFile) (interface{}, error) { 909 file, err := f.Open() 910 if err != nil { 911 return nil, fmt.Errorf("readData: failed to open data file: %s", err) 912 } 913 defer file.Close() 914 content := helpers.ReaderToBytes(file) 915 916 switch f.Extension() { 917 case "yaml", "yml": 918 return parser.HandleYAMLData(content) 919 case "json": 920 return parser.HandleJSONData(content) 921 case "toml": 922 return parser.HandleTOMLMetaData(content) 923 default: 924 return nil, fmt.Errorf("Data not supported for extension '%s'", f.Extension()) 925 } 926 } 927 928 func (s *Site) readDataFromSourceFS() error { 929 err := s.loadData(s.PathSpec.BaseFs.Data.Fs) 930 s.timerStep("load data") 931 return err 932 } 933 934 func (s *Site) process(config BuildCfg) (err error) { 935 if err = s.initialize(); err != nil { 936 return 937 } 938 s.timerStep("initialize") 939 940 if err = s.readDataFromSourceFS(); err != nil { 941 return 942 } 943 944 s.timerStep("load i18n") 945 946 if err := s.readAndProcessContent(); err != nil { 947 return err 948 } 949 s.timerStep("read and convert pages from source") 950 951 return err 952 953 } 954 955 func (s *Site) setupSitePages() { 956 var siteLastChange time.Time 957 958 for i, page := range s.RegularPages { 959 if i < len(s.RegularPages)-1 { 960 page.Next = s.RegularPages[i+1] 961 } 962 963 if i > 0 { 964 page.Prev = s.RegularPages[i-1] 965 } 966 967 // Determine Site.Info.LastChange 968 // Note that the logic to determine which date to use for Lastmod 969 // is already applied, so this is *the* date to use. 970 // We cannot just pick the last page in the default sort, because 971 // that may not be ordered by date. 972 if page.Lastmod.After(siteLastChange) { 973 siteLastChange = page.Lastmod 974 } 975 } 976 977 s.Info.LastChange = siteLastChange 978 } 979 980 func (s *Site) render(config *BuildCfg, outFormatIdx int) (err error) { 981 982 if outFormatIdx == 0 { 983 if err = s.preparePages(); err != nil { 984 return 985 } 986 s.timerStep("prepare pages") 987 988 // Note that even if disableAliases is set, the aliases themselves are 989 // preserved on page. The motivation with this is to be able to generate 990 // 301 redirects in a .htacess file and similar using a custom output format. 991 if !s.Cfg.GetBool("disableAliases") { 992 // Aliases must be rendered before pages. 993 // Some sites, Hugo docs included, have faulty alias definitions that point 994 // to itself or another real page. These will be overwritten in the next 995 // step. 996 if err = s.renderAliases(); err != nil { 997 return 998 } 999 s.timerStep("render and write aliases") 1000 } 1001 1002 } 1003 1004 if err = s.renderPages(config); err != nil { 1005 return 1006 } 1007 1008 s.timerStep("render and write pages") 1009 1010 // TODO(bep) render consider this, ref. render404 etc. 1011 if outFormatIdx > 0 { 1012 return 1013 } 1014 1015 if err = s.renderSitemap(); err != nil { 1016 return 1017 } 1018 s.timerStep("render and write Sitemap") 1019 1020 if err = s.renderRobotsTXT(); err != nil { 1021 return 1022 } 1023 s.timerStep("render and write robots.txt") 1024 1025 if err = s.render404(); err != nil { 1026 return 1027 } 1028 s.timerStep("render and write 404") 1029 1030 return 1031 } 1032 1033 func (s *Site) Initialise() (err error) { 1034 return s.initialize() 1035 } 1036 1037 func (s *Site) initialize() (err error) { 1038 s.Menus = Menus{} 1039 1040 return s.initializeSiteInfo() 1041 } 1042 1043 // HomeAbsURL is a convenience method giving the absolute URL to the home page. 1044 func (s *SiteInfo) HomeAbsURL() string { 1045 base := "" 1046 if s.IsMultiLingual() { 1047 base = s.Language.Lang 1048 } 1049 return s.owner.AbsURL(base, false) 1050 } 1051 1052 // SitemapAbsURL is a convenience method giving the absolute URL to the sitemap. 1053 func (s *SiteInfo) SitemapAbsURL() string { 1054 sitemapDefault := parseSitemap(s.s.Cfg.GetStringMap("sitemap")) 1055 p := s.HomeAbsURL() 1056 if !strings.HasSuffix(p, "/") { 1057 p += "/" 1058 } 1059 p += sitemapDefault.Filename 1060 return p 1061 } 1062 1063 func (s *Site) initializeSiteInfo() error { 1064 var ( 1065 lang = s.Language 1066 languages langs.Languages 1067 ) 1068 1069 if s.owner != nil && s.owner.multilingual != nil { 1070 languages = s.owner.multilingual.Languages 1071 } 1072 1073 params := lang.Params() 1074 1075 permalinks := make(PermalinkOverrides) 1076 for k, v := range s.Cfg.GetStringMapString("permalinks") { 1077 permalinks[k] = pathPattern(v) 1078 } 1079 1080 defaultContentInSubDir := s.Cfg.GetBool("defaultContentLanguageInSubdir") 1081 defaultContentLanguage := s.Cfg.GetString("defaultContentLanguage") 1082 1083 languagePrefix := "" 1084 if s.multilingualEnabled() && (defaultContentInSubDir || lang.Lang != defaultContentLanguage) { 1085 languagePrefix = "/" + lang.Lang 1086 } 1087 1088 var multilingual *Multilingual 1089 if s.owner != nil { 1090 multilingual = s.owner.multilingual 1091 } 1092 1093 var uglyURLs = func(p *Page) bool { 1094 return false 1095 } 1096 1097 v := s.Cfg.Get("uglyURLs") 1098 if v != nil { 1099 switch vv := v.(type) { 1100 case bool: 1101 uglyURLs = func(p *Page) bool { 1102 return vv 1103 } 1104 case string: 1105 // Is what be get from CLI (--uglyURLs) 1106 vvv := cast.ToBool(vv) 1107 uglyURLs = func(p *Page) bool { 1108 return vvv 1109 } 1110 default: 1111 m := cast.ToStringMapBool(v) 1112 uglyURLs = func(p *Page) bool { 1113 return m[p.Section()] 1114 } 1115 } 1116 } 1117 1118 siteConfig, err := loadSiteConfig(lang) 1119 if err != nil { 1120 return err 1121 } 1122 1123 s.Info = SiteInfo{ 1124 Title: lang.GetString("title"), 1125 Author: lang.GetStringMap("author"), 1126 Social: lang.GetStringMapString("social"), 1127 LanguageCode: lang.GetString("languageCode"), 1128 Copyright: lang.GetString("copyright"), 1129 multilingual: multilingual, 1130 Language: lang, 1131 LanguagePrefix: languagePrefix, 1132 Languages: languages, 1133 defaultContentLanguageInSubdir: defaultContentInSubDir, 1134 sectionPagesMenu: lang.GetString("sectionPagesMenu"), 1135 BuildDrafts: s.Cfg.GetBool("buildDrafts"), 1136 canonifyURLs: s.Cfg.GetBool("canonifyURLs"), 1137 relativeURLs: s.Cfg.GetBool("relativeURLs"), 1138 uglyURLs: uglyURLs, 1139 preserveTaxonomyNames: lang.GetBool("preserveTaxonomyNames"), 1140 PageCollections: s.PageCollections, 1141 Menus: &s.Menus, 1142 Params: params, 1143 Permalinks: permalinks, 1144 Data: &s.Data, 1145 owner: s.owner, 1146 s: s, 1147 Config: siteConfig, 1148 // TODO(bep) make this Menu and similar into delegate methods on SiteInfo 1149 Taxonomies: s.Taxonomies, 1150 } 1151 1152 rssOutputFormat, found := s.outputFormats[KindHome].GetByName(output.RSSFormat.Name) 1153 1154 if found { 1155 s.Info.RSSLink = s.permalink(rssOutputFormat.BaseFilename()) 1156 } 1157 1158 return nil 1159 } 1160 1161 func (s *Site) isI18nEvent(e fsnotify.Event) bool { 1162 return s.BaseFs.SourceFilesystems.IsI18n(e.Name) 1163 } 1164 1165 func (s *Site) isDataDirEvent(e fsnotify.Event) bool { 1166 return s.BaseFs.SourceFilesystems.IsData(e.Name) 1167 } 1168 1169 func (s *Site) isLayoutDirEvent(e fsnotify.Event) bool { 1170 return s.BaseFs.SourceFilesystems.IsLayout(e.Name) 1171 } 1172 1173 func (s *Site) absContentDir() string { 1174 return s.PathSpec.AbsPathify(s.PathSpec.ContentDir) 1175 } 1176 1177 func (s *Site) isContentDirEvent(e fsnotify.Event) bool { 1178 return s.BaseFs.IsContent(e.Name) 1179 } 1180 1181 type contentCaptureResultHandler struct { 1182 defaultContentProcessor *siteContentProcessor 1183 contentProcessors map[string]*siteContentProcessor 1184 } 1185 1186 func (c *contentCaptureResultHandler) getContentProcessor(lang string) *siteContentProcessor { 1187 proc, found := c.contentProcessors[lang] 1188 if found { 1189 return proc 1190 } 1191 return c.defaultContentProcessor 1192 } 1193 1194 func (c *contentCaptureResultHandler) handleSingles(fis ...*fileInfo) { 1195 for _, fi := range fis { 1196 proc := c.getContentProcessor(fi.Lang()) 1197 proc.processSingle(fi) 1198 } 1199 } 1200 func (c *contentCaptureResultHandler) handleBundles(d *bundleDirs) { 1201 for _, b := range d.bundles { 1202 proc := c.getContentProcessor(b.fi.Lang()) 1203 proc.processBundle(b) 1204 } 1205 } 1206 1207 func (c *contentCaptureResultHandler) handleCopyFiles(files ...pathLangFile) { 1208 for _, proc := range c.contentProcessors { 1209 proc.processAssets(files) 1210 } 1211 } 1212 1213 func (s *Site) readAndProcessContent(filenames ...string) error { 1214 ctx := context.Background() 1215 g, ctx := errgroup.WithContext(ctx) 1216 1217 defaultContentLanguage := s.SourceSpec.DefaultContentLanguage 1218 1219 contentProcessors := make(map[string]*siteContentProcessor) 1220 var defaultContentProcessor *siteContentProcessor 1221 sites := s.owner.langSite() 1222 for k, v := range sites { 1223 if v.Language.Disabled { 1224 continue 1225 } 1226 proc := newSiteContentProcessor(ctx, len(filenames) > 0, v) 1227 contentProcessors[k] = proc 1228 if k == defaultContentLanguage { 1229 defaultContentProcessor = proc 1230 } 1231 g.Go(func() error { 1232 return proc.process(ctx) 1233 }) 1234 } 1235 1236 var ( 1237 handler captureResultHandler 1238 bundleMap *contentChangeMap 1239 ) 1240 1241 mainHandler := &contentCaptureResultHandler{contentProcessors: contentProcessors, defaultContentProcessor: defaultContentProcessor} 1242 1243 sourceSpec := source.NewSourceSpec(s.PathSpec, s.BaseFs.ContentFs) 1244 1245 if s.running() { 1246 // Need to track changes. 1247 bundleMap = s.owner.ContentChanges 1248 handler = &captureResultHandlerChain{handlers: []captureBundlesHandler{mainHandler, bundleMap}} 1249 1250 } else { 1251 handler = mainHandler 1252 } 1253 1254 c := newCapturer(s.Log, sourceSpec, handler, bundleMap, filenames...) 1255 1256 err1 := c.capture() 1257 1258 for _, proc := range contentProcessors { 1259 proc.closeInput() 1260 } 1261 1262 err2 := g.Wait() 1263 1264 if err1 != nil { 1265 return err1 1266 } 1267 return err2 1268 } 1269 1270 func (s *Site) buildSiteMeta() (err error) { 1271 defer s.timerStep("build Site meta") 1272 1273 if len(s.Pages) == 0 { 1274 return 1275 } 1276 1277 s.assembleTaxonomies() 1278 1279 for _, p := range s.AllPages { 1280 // this depends on taxonomies 1281 p.setValuesForKind(s) 1282 } 1283 1284 return 1285 } 1286 1287 func (s *Site) getMenusFromConfig() Menus { 1288 1289 ret := Menus{} 1290 1291 if menus := s.Language.GetStringMap("menu"); menus != nil { 1292 for name, menu := range menus { 1293 m, err := cast.ToSliceE(menu) 1294 if err != nil { 1295 s.Log.ERROR.Printf("unable to process menus in site config\n") 1296 s.Log.ERROR.Println(err) 1297 } else { 1298 for _, entry := range m { 1299 s.Log.DEBUG.Printf("found menu: %q, in site config\n", name) 1300 1301 menuEntry := MenuEntry{Menu: name} 1302 ime, err := cast.ToStringMapE(entry) 1303 if err != nil { 1304 s.Log.ERROR.Printf("unable to process menus in site config\n") 1305 s.Log.ERROR.Println(err) 1306 } 1307 1308 menuEntry.marshallMap(ime) 1309 menuEntry.URL = s.Info.createNodeMenuEntryURL(menuEntry.URL) 1310 1311 if ret[name] == nil { 1312 ret[name] = &Menu{} 1313 } 1314 *ret[name] = ret[name].add(&menuEntry) 1315 } 1316 } 1317 } 1318 return ret 1319 } 1320 return ret 1321 } 1322 1323 func (s *SiteInfo) createNodeMenuEntryURL(in string) string { 1324 1325 if !strings.HasPrefix(in, "/") { 1326 return in 1327 } 1328 // make it match the nodes 1329 menuEntryURL := in 1330 menuEntryURL = helpers.SanitizeURLKeepTrailingSlash(s.s.PathSpec.URLize(menuEntryURL)) 1331 if !s.canonifyURLs { 1332 menuEntryURL = helpers.AddContextRoot(s.s.PathSpec.BaseURL.String(), menuEntryURL) 1333 } 1334 return menuEntryURL 1335 } 1336 1337 func (s *Site) assembleMenus() { 1338 s.Menus = Menus{} 1339 1340 type twoD struct { 1341 MenuName, EntryName string 1342 } 1343 flat := map[twoD]*MenuEntry{} 1344 children := map[twoD]Menu{} 1345 1346 // add menu entries from config to flat hash 1347 menuConfig := s.getMenusFromConfig() 1348 for name, menu := range menuConfig { 1349 for _, me := range *menu { 1350 flat[twoD{name, me.KeyName()}] = me 1351 } 1352 } 1353 1354 sectionPagesMenu := s.Info.sectionPagesMenu 1355 pages := s.Pages 1356 1357 if sectionPagesMenu != "" { 1358 for _, p := range pages { 1359 if p.Kind == KindSection { 1360 // From Hugo 0.22 we have nested sections, but until we get a 1361 // feel of how that would work in this setting, let us keep 1362 // this menu for the top level only. 1363 id := p.Section() 1364 if _, ok := flat[twoD{sectionPagesMenu, id}]; ok { 1365 continue 1366 } 1367 1368 me := MenuEntry{Identifier: id, 1369 Name: p.LinkTitle(), 1370 Weight: p.Weight, 1371 URL: p.RelPermalink()} 1372 flat[twoD{sectionPagesMenu, me.KeyName()}] = &me 1373 } 1374 } 1375 } 1376 1377 // Add menu entries provided by pages 1378 for _, p := range pages { 1379 for name, me := range p.Menus() { 1380 if _, ok := flat[twoD{name, me.KeyName()}]; ok { 1381 s.Log.ERROR.Printf("Two or more menu items have the same name/identifier in Menu %q: %q.\nRename or set an unique identifier.\n", name, me.KeyName()) 1382 continue 1383 } 1384 flat[twoD{name, me.KeyName()}] = me 1385 } 1386 } 1387 1388 // Create Children Menus First 1389 for _, e := range flat { 1390 if e.Parent != "" { 1391 children[twoD{e.Menu, e.Parent}] = children[twoD{e.Menu, e.Parent}].add(e) 1392 } 1393 } 1394 1395 // Placing Children in Parents (in flat) 1396 for p, childmenu := range children { 1397 _, ok := flat[twoD{p.MenuName, p.EntryName}] 1398 if !ok { 1399 // if parent does not exist, create one without a URL 1400 flat[twoD{p.MenuName, p.EntryName}] = &MenuEntry{Name: p.EntryName, URL: ""} 1401 } 1402 flat[twoD{p.MenuName, p.EntryName}].Children = childmenu 1403 } 1404 1405 // Assembling Top Level of Tree 1406 for menu, e := range flat { 1407 if e.Parent == "" { 1408 _, ok := s.Menus[menu.MenuName] 1409 if !ok { 1410 s.Menus[menu.MenuName] = &Menu{} 1411 } 1412 *s.Menus[menu.MenuName] = s.Menus[menu.MenuName].add(e) 1413 } 1414 } 1415 } 1416 1417 func (s *Site) getTaxonomyKey(key string) string { 1418 if s.Info.preserveTaxonomyNames { 1419 // Keep as is 1420 return key 1421 } 1422 return s.PathSpec.MakePathSanitized(key) 1423 } 1424 1425 // We need to create the top level taxonomy early in the build process 1426 // to be able to determine the page Kind correctly. 1427 func (s *Site) createTaxonomiesEntries() { 1428 s.Taxonomies = make(TaxonomyList) 1429 taxonomies := s.Language.GetStringMapString("taxonomies") 1430 for _, plural := range taxonomies { 1431 s.Taxonomies[plural] = make(Taxonomy) 1432 } 1433 } 1434 1435 func (s *Site) assembleTaxonomies() { 1436 s.taxonomiesPluralSingular = make(map[string]string) 1437 s.taxonomiesOrigKey = make(map[string]string) 1438 1439 taxonomies := s.Language.GetStringMapString("taxonomies") 1440 1441 s.Log.INFO.Printf("found taxonomies: %#v\n", taxonomies) 1442 1443 for singular, plural := range taxonomies { 1444 s.taxonomiesPluralSingular[plural] = singular 1445 1446 for _, p := range s.Pages { 1447 vals := p.getParam(plural, !s.Info.preserveTaxonomyNames) 1448 weight := p.getParamToLower(plural + "_weight") 1449 if weight == nil { 1450 weight = 0 1451 } 1452 if vals != nil { 1453 if v, ok := vals.([]string); ok { 1454 for _, idx := range v { 1455 x := WeightedPage{weight.(int), p} 1456 s.Taxonomies[plural].add(s.getTaxonomyKey(idx), x) 1457 if s.Info.preserveTaxonomyNames { 1458 // Need to track the original 1459 s.taxonomiesOrigKey[fmt.Sprintf("%s-%s", plural, s.PathSpec.MakePathSanitized(idx))] = idx 1460 } 1461 } 1462 } else if v, ok := vals.(string); ok { 1463 x := WeightedPage{weight.(int), p} 1464 s.Taxonomies[plural].add(s.getTaxonomyKey(v), x) 1465 if s.Info.preserveTaxonomyNames { 1466 // Need to track the original 1467 s.taxonomiesOrigKey[fmt.Sprintf("%s-%s", plural, s.PathSpec.MakePathSanitized(v))] = v 1468 } 1469 } else { 1470 s.Log.ERROR.Printf("Invalid %s in %s\n", plural, p.File.Path()) 1471 } 1472 } 1473 } 1474 for k := range s.Taxonomies[plural] { 1475 s.Taxonomies[plural][k].Sort() 1476 } 1477 } 1478 1479 s.Info.Taxonomies = s.Taxonomies 1480 } 1481 1482 // Prepare site for a new full build. 1483 func (s *Site) resetBuildState() { 1484 1485 s.relatedDocsHandler = newSearchIndexHandler(s.relatedDocsHandler.cfg) 1486 s.PageCollections = newPageCollectionsFromPages(s.rawAllPages) 1487 // TODO(bep) get rid of this double 1488 s.Info.PageCollections = s.PageCollections 1489 1490 s.draftCount = 0 1491 s.futureCount = 0 1492 1493 s.expiredCount = 0 1494 1495 spc = newPageCache() 1496 1497 for _, p := range s.rawAllPages { 1498 p.subSections = Pages{} 1499 p.parent = nil 1500 p.scratch = newScratch() 1501 p.mainPageOutput = nil 1502 } 1503 } 1504 1505 func (s *Site) kindFromSections(sections []string) string { 1506 if len(sections) == 0 { 1507 return KindSection 1508 } 1509 1510 if _, isTaxonomy := s.Taxonomies[sections[0]]; isTaxonomy { 1511 if len(sections) == 1 { 1512 return KindTaxonomyTerm 1513 } 1514 return KindTaxonomy 1515 } 1516 return KindSection 1517 } 1518 1519 func (s *Site) layouts(p *PageOutput) ([]string, error) { 1520 return s.layoutHandler.For(p.layoutDescriptor, p.outputFormat) 1521 } 1522 1523 func (s *Site) preparePages() error { 1524 var errors []error 1525 1526 for _, p := range s.Pages { 1527 if err := p.prepareLayouts(); err != nil { 1528 errors = append(errors, err) 1529 } 1530 if err := p.prepareData(s); err != nil { 1531 errors = append(errors, err) 1532 } 1533 } 1534 1535 if len(errors) != 0 { 1536 return fmt.Errorf("Prepare pages failed: %.100q…", errors) 1537 } 1538 1539 return nil 1540 } 1541 1542 func errorCollator(results <-chan error, errs chan<- error) { 1543 errMsgs := []string{} 1544 for err := range results { 1545 if err != nil { 1546 errMsgs = append(errMsgs, err.Error()) 1547 } 1548 } 1549 if len(errMsgs) == 0 { 1550 errs <- nil 1551 } else { 1552 errs <- errors.New(strings.Join(errMsgs, "\n")) 1553 } 1554 close(errs) 1555 } 1556 1557 func (s *Site) appendThemeTemplates(in []string) []string { 1558 if !s.PathSpec.ThemeSet() { 1559 return in 1560 } 1561 1562 out := []string{} 1563 // First place all non internal templates 1564 for _, t := range in { 1565 if !strings.HasPrefix(t, "_internal/") { 1566 out = append(out, t) 1567 } 1568 } 1569 1570 // Then place theme templates with the same names 1571 for _, t := range in { 1572 if !strings.HasPrefix(t, "_internal/") { 1573 out = append(out, "theme/"+t) 1574 } 1575 } 1576 1577 // Lastly place internal templates 1578 for _, t := range in { 1579 if strings.HasPrefix(t, "_internal/") { 1580 out = append(out, t) 1581 } 1582 } 1583 return out 1584 1585 } 1586 1587 // GetPage looks up a page of a given type in the path given. 1588 // {{ with .Site.GetPage "section" "blog" }}{{ .Title }}{{ end }} 1589 // 1590 // This will return nil when no page could be found, and will return the 1591 // first page found if the key is ambigous. 1592 func (s *SiteInfo) GetPage(typ string, path ...string) (*Page, error) { 1593 return s.getPage(typ, path...), nil 1594 } 1595 1596 func (s *Site) permalinkForOutputFormat(link string, f output.Format) (string, error) { 1597 var ( 1598 baseURL string 1599 err error 1600 ) 1601 1602 if f.Protocol != "" { 1603 baseURL, err = s.PathSpec.BaseURL.WithProtocol(f.Protocol) 1604 if err != nil { 1605 return "", err 1606 } 1607 } else { 1608 baseURL = s.PathSpec.BaseURL.String() 1609 } 1610 return s.PathSpec.PermalinkForBaseURL(link, baseURL), nil 1611 } 1612 1613 func (s *Site) permalink(link string) string { 1614 return s.PathSpec.PermalinkForBaseURL(link, s.PathSpec.BaseURL.String()) 1615 1616 } 1617 1618 func (s *Site) renderAndWriteXML(statCounter *uint64, name string, dest string, d interface{}, layouts ...string) error { 1619 s.Log.DEBUG.Printf("Render XML for %q to %q", name, dest) 1620 renderBuffer := bp.GetBuffer() 1621 defer bp.PutBuffer(renderBuffer) 1622 renderBuffer.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?>\n") 1623 1624 if err := s.renderForLayouts(name, d, renderBuffer, layouts...); err != nil { 1625 helpers.DistinctWarnLog.Println(err) 1626 return nil 1627 } 1628 1629 outBuffer := bp.GetBuffer() 1630 defer bp.PutBuffer(outBuffer) 1631 1632 var path []byte 1633 if s.Info.relativeURLs { 1634 path = []byte(helpers.GetDottedRelativePath(dest)) 1635 } else { 1636 s := s.PathSpec.BaseURL.String() 1637 if !strings.HasSuffix(s, "/") { 1638 s += "/" 1639 } 1640 path = []byte(s) 1641 } 1642 transformer := transform.NewChain(transform.AbsURLInXML) 1643 if err := transformer.Apply(outBuffer, renderBuffer, path); err != nil { 1644 s.DistinctErrorLog.Println(err) 1645 return nil 1646 } 1647 1648 return s.publish(statCounter, dest, outBuffer) 1649 1650 } 1651 1652 func (s *Site) renderAndWritePage(statCounter *uint64, name string, dest string, p *PageOutput, layouts ...string) error { 1653 renderBuffer := bp.GetBuffer() 1654 defer bp.PutBuffer(renderBuffer) 1655 1656 if err := s.renderForLayouts(p.Kind, p, renderBuffer, layouts...); err != nil { 1657 helpers.DistinctWarnLog.Println(err) 1658 return nil 1659 } 1660 1661 if renderBuffer.Len() == 0 { 1662 return nil 1663 } 1664 1665 outBuffer := bp.GetBuffer() 1666 defer bp.PutBuffer(outBuffer) 1667 1668 transformLinks := transform.NewEmptyTransforms() 1669 1670 isHTML := p.outputFormat.IsHTML 1671 1672 if isHTML { 1673 if s.Info.relativeURLs || s.Info.canonifyURLs { 1674 transformLinks = append(transformLinks, transform.AbsURL) 1675 } 1676 1677 if s.running() && s.Cfg.GetBool("watch") && !s.Cfg.GetBool("disableLiveReload") { 1678 transformLinks = append(transformLinks, transform.LiveReloadInject(s.Cfg.GetInt("liveReloadPort"))) 1679 } 1680 1681 // For performance reasons we only inject the Hugo generator tag on the home page. 1682 if p.IsHome() { 1683 if !s.Cfg.GetBool("disableHugoGeneratorInject") { 1684 transformLinks = append(transformLinks, transform.HugoGeneratorInject) 1685 } 1686 } 1687 } 1688 1689 var path []byte 1690 1691 if s.Info.relativeURLs { 1692 path = []byte(helpers.GetDottedRelativePath(dest)) 1693 } else if s.Info.canonifyURLs { 1694 url := s.PathSpec.BaseURL.String() 1695 if !strings.HasSuffix(url, "/") { 1696 url += "/" 1697 } 1698 path = []byte(url) 1699 } 1700 1701 transformer := transform.NewChain(transformLinks...) 1702 if err := transformer.Apply(outBuffer, renderBuffer, path); err != nil { 1703 s.DistinctErrorLog.Println(err) 1704 return nil 1705 } 1706 1707 return s.publish(statCounter, dest, outBuffer) 1708 } 1709 1710 func (s *Site) renderForLayouts(name string, d interface{}, w io.Writer, layouts ...string) (err error) { 1711 var templ tpl.Template 1712 1713 defer func() { 1714 if r := recover(); r != nil { 1715 templName := "" 1716 if templ != nil { 1717 templName = templ.Name() 1718 } 1719 s.DistinctErrorLog.Printf("Failed to render %q: %s", templName, r) 1720 // TOD(bep) we really need to fix this. Also see below. 1721 if !s.running() && !testMode { 1722 os.Exit(-1) 1723 } 1724 } 1725 }() 1726 1727 templ = s.findFirstTemplate(layouts...) 1728 if templ == nil { 1729 return fmt.Errorf("[%s] Unable to locate layout for %q: %s\n", s.Language.Lang, name, layouts) 1730 } 1731 1732 if err = templ.Execute(w, d); err != nil { 1733 // Behavior here should be dependent on if running in server or watch mode. 1734 if p, ok := d.(*PageOutput); ok { 1735 if p.File != nil { 1736 s.DistinctErrorLog.Printf("Error while rendering %q in %q: %s", name, p.File.Dir(), err) 1737 } else { 1738 s.DistinctErrorLog.Printf("Error while rendering %q: %s", name, err) 1739 } 1740 } else { 1741 s.DistinctErrorLog.Printf("Error while rendering %q: %s", name, err) 1742 } 1743 if !s.running() && !testMode { 1744 // TODO(bep) check if this can be propagated 1745 os.Exit(-1) 1746 } else if testMode { 1747 return 1748 } 1749 } 1750 1751 return 1752 } 1753 1754 func (s *Site) findFirstTemplate(layouts ...string) tpl.Template { 1755 for _, layout := range layouts { 1756 if templ := s.Tmpl.Lookup(layout); templ != nil { 1757 return templ 1758 } 1759 } 1760 return nil 1761 } 1762 1763 func (s *Site) publish(statCounter *uint64, path string, r io.Reader) (err error) { 1764 s.PathSpec.ProcessingStats.Incr(statCounter) 1765 1766 return helpers.WriteToDisk(filepath.Clean(path), r, s.BaseFs.PublishFs) 1767 } 1768 1769 func getGoMaxProcs() int { 1770 if gmp := os.Getenv("GOMAXPROCS"); gmp != "" { 1771 if p, err := strconv.Atoi(gmp); err != nil { 1772 return p 1773 } 1774 } 1775 return 1 1776 } 1777 1778 func (s *Site) newNodePage(typ string, sections ...string) *Page { 1779 p := &Page{ 1780 language: s.Language, 1781 pageInit: &pageInit{}, 1782 pageContentInit: &pageContentInit{}, 1783 Kind: typ, 1784 Source: Source{File: &source.FileInfo{}}, 1785 Data: make(map[string]interface{}), 1786 Site: &s.Info, 1787 sections: sections, 1788 s: s} 1789 1790 p.outputFormats = p.s.outputFormats[p.Kind] 1791 1792 return p 1793 1794 } 1795 1796 func (s *Site) newHomePage() *Page { 1797 p := s.newNodePage(KindHome) 1798 p.title = s.Info.Title 1799 pages := Pages{} 1800 p.Data["Pages"] = pages 1801 p.Pages = pages 1802 return p 1803 } 1804 1805 func (s *Site) newTaxonomyPage(plural, key string) *Page { 1806 1807 p := s.newNodePage(KindTaxonomy, plural, key) 1808 1809 if s.Info.preserveTaxonomyNames { 1810 // Keep (mostly) as is in the title 1811 // We make the first character upper case, mostly because 1812 // it is easier to reason about in the tests. 1813 p.title = helpers.FirstUpper(key) 1814 key = s.PathSpec.MakePathSanitized(key) 1815 } else { 1816 p.title = strings.Replace(s.titleFunc(key), "-", " ", -1) 1817 } 1818 1819 return p 1820 } 1821 1822 func (s *Site) newSectionPage(name string) *Page { 1823 p := s.newNodePage(KindSection, name) 1824 1825 sectionName := helpers.FirstUpper(name) 1826 if s.Cfg.GetBool("pluralizeListTitles") { 1827 p.title = inflect.Pluralize(sectionName) 1828 } else { 1829 p.title = sectionName 1830 } 1831 return p 1832 } 1833 1834 func (s *Site) newTaxonomyTermsPage(plural string) *Page { 1835 p := s.newNodePage(KindTaxonomyTerm, plural) 1836 p.title = s.titleFunc(plural) 1837 return p 1838 }