github.com/kovansky/hugo@v0.92.3-0.20220224232819-63076e4ff19f/hugolib/hugo_sites.go (about)

     1  // Copyright 2019 The Hugo Authors. All rights reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  // http://www.apache.org/licenses/LICENSE-2.0
     7  //
     8  // Unless required by applicable law or agreed to in writing, software
     9  // distributed under the License is distributed on an "AS IS" BASIS,
    10  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package hugolib
    15  
    16  import (
    17  	"context"
    18  	"io"
    19  	"path/filepath"
    20  	"sort"
    21  	"strings"
    22  	"sync"
    23  	"sync/atomic"
    24  
    25  	"github.com/gohugoio/hugo/hugofs/glob"
    26  
    27  	"github.com/fsnotify/fsnotify"
    28  
    29  	"github.com/gohugoio/hugo/identity"
    30  
    31  	radix "github.com/armon/go-radix"
    32  
    33  	"github.com/gohugoio/hugo/output"
    34  	"github.com/gohugoio/hugo/parser/metadecoders"
    35  
    36  	"github.com/gohugoio/hugo/common/para"
    37  	"github.com/gohugoio/hugo/hugofs"
    38  	"github.com/pkg/errors"
    39  
    40  	"github.com/gohugoio/hugo/source"
    41  
    42  	"github.com/bep/gitmap"
    43  	"github.com/gohugoio/hugo/config"
    44  
    45  	"github.com/gohugoio/hugo/publisher"
    46  
    47  	"github.com/gohugoio/hugo/common/herrors"
    48  	"github.com/gohugoio/hugo/common/loggers"
    49  	"github.com/gohugoio/hugo/deps"
    50  	"github.com/gohugoio/hugo/helpers"
    51  	"github.com/gohugoio/hugo/langs"
    52  	"github.com/gohugoio/hugo/lazy"
    53  
    54  	"github.com/gohugoio/hugo/langs/i18n"
    55  	"github.com/gohugoio/hugo/resources/page"
    56  	"github.com/gohugoio/hugo/resources/page/pagemeta"
    57  	"github.com/gohugoio/hugo/tpl"
    58  	"github.com/gohugoio/hugo/tpl/tplimpl"
    59  )
    60  
    61  // HugoSites represents the sites to build. Each site represents a language.
    62  type HugoSites struct {
    63  	Sites []*Site
    64  
    65  	multilingual *Multilingual
    66  
    67  	// Multihost is set if multilingual and baseURL set on the language level.
    68  	multihost bool
    69  
    70  	// If this is running in the dev server.
    71  	running bool
    72  
    73  	// Render output formats for all sites.
    74  	renderFormats output.Formats
    75  
    76  	*deps.Deps
    77  
    78  	gitInfo       *gitInfo
    79  	codeownerInfo *codeownerInfo
    80  
    81  	// As loaded from the /data dirs
    82  	data map[string]interface{}
    83  
    84  	contentInit sync.Once
    85  	content     *pageMaps
    86  
    87  	// Keeps track of bundle directories and symlinks to enable partial rebuilding.
    88  	ContentChanges *contentChangeMap
    89  
    90  	// File change events with filename stored in this map will be skipped.
    91  	skipRebuildForFilenamesMu sync.Mutex
    92  	skipRebuildForFilenames   map[string]bool
    93  
    94  	init *hugoSitesInit
    95  
    96  	workers    *para.Workers
    97  	numWorkers int
    98  
    99  	*fatalErrorHandler
   100  	*testCounters
   101  }
   102  
   103  // ShouldSkipFileChangeEvent allows skipping filesystem event early before
   104  // the build is started.
   105  func (h *HugoSites) ShouldSkipFileChangeEvent(ev fsnotify.Event) bool {
   106  	h.skipRebuildForFilenamesMu.Lock()
   107  	defer h.skipRebuildForFilenamesMu.Unlock()
   108  	return h.skipRebuildForFilenames[ev.Name]
   109  }
   110  
   111  func (h *HugoSites) getContentMaps() *pageMaps {
   112  	h.contentInit.Do(func() {
   113  		h.content = newPageMaps(h)
   114  	})
   115  	return h.content
   116  }
   117  
   118  // Only used in tests.
   119  type testCounters struct {
   120  	contentRenderCounter uint64
   121  	pageRenderCounter    uint64
   122  }
   123  
   124  func (h *testCounters) IncrContentRender() {
   125  	if h == nil {
   126  		return
   127  	}
   128  	atomic.AddUint64(&h.contentRenderCounter, 1)
   129  }
   130  
   131  func (h *testCounters) IncrPageRender() {
   132  	if h == nil {
   133  		return
   134  	}
   135  	atomic.AddUint64(&h.pageRenderCounter, 1)
   136  }
   137  
   138  type fatalErrorHandler struct {
   139  	mu sync.Mutex
   140  
   141  	h *HugoSites
   142  
   143  	err error
   144  
   145  	done  bool
   146  	donec chan bool // will be closed when done
   147  }
   148  
   149  // FatalError error is used in some rare situations where it does not make sense to
   150  // continue processing, to abort as soon as possible and log the error.
   151  func (f *fatalErrorHandler) FatalError(err error) {
   152  	f.mu.Lock()
   153  	defer f.mu.Unlock()
   154  	if !f.done {
   155  		f.done = true
   156  		close(f.donec)
   157  	}
   158  	f.err = err
   159  }
   160  
   161  func (f *fatalErrorHandler) getErr() error {
   162  	f.mu.Lock()
   163  	defer f.mu.Unlock()
   164  	return f.err
   165  }
   166  
   167  func (f *fatalErrorHandler) Done() <-chan bool {
   168  	return f.donec
   169  }
   170  
   171  type hugoSitesInit struct {
   172  	// Loads the data from all of the /data folders.
   173  	data *lazy.Init
   174  
   175  	// Performs late initialization (before render) of the templates.
   176  	layouts *lazy.Init
   177  
   178  	// Loads the Git info and CODEOWNERS for all the pages if enabled.
   179  	gitInfo *lazy.Init
   180  
   181  	// Maps page translations.
   182  	translations *lazy.Init
   183  }
   184  
   185  func (h *hugoSitesInit) Reset() {
   186  	h.data.Reset()
   187  	h.layouts.Reset()
   188  	h.gitInfo.Reset()
   189  	h.translations.Reset()
   190  }
   191  
   192  func (h *HugoSites) Data() map[string]interface{} {
   193  	if _, err := h.init.data.Do(); err != nil {
   194  		h.SendError(errors.Wrap(err, "failed to load data"))
   195  		return nil
   196  	}
   197  	return h.data
   198  }
   199  
   200  func (h *HugoSites) gitInfoForPage(p page.Page) (*gitmap.GitInfo, error) {
   201  	if _, err := h.init.gitInfo.Do(); err != nil {
   202  		return nil, err
   203  	}
   204  
   205  	if h.gitInfo == nil {
   206  		return nil, nil
   207  	}
   208  
   209  	return h.gitInfo.forPage(p), nil
   210  }
   211  
   212  func (h *HugoSites) codeownersForPage(p page.Page) ([]string, error) {
   213  	if _, err := h.init.gitInfo.Do(); err != nil {
   214  		return nil, err
   215  	}
   216  
   217  	if h.codeownerInfo == nil {
   218  		return nil, nil
   219  	}
   220  
   221  	return h.codeownerInfo.forPage(p), nil
   222  }
   223  
   224  func (h *HugoSites) siteInfos() page.Sites {
   225  	infos := make(page.Sites, len(h.Sites))
   226  	for i, site := range h.Sites {
   227  		infos[i] = site.Info
   228  	}
   229  	return infos
   230  }
   231  
   232  func (h *HugoSites) pickOneAndLogTheRest(errors []error) error {
   233  	if len(errors) == 0 {
   234  		return nil
   235  	}
   236  
   237  	var i int
   238  
   239  	for j, err := range errors {
   240  		// If this is in server mode, we want to return an error to the client
   241  		// with a file context, if possible.
   242  		if herrors.UnwrapErrorWithFileContext(err) != nil {
   243  			i = j
   244  			break
   245  		}
   246  	}
   247  
   248  	// Log the rest, but add a threshold to avoid flooding the log.
   249  	const errLogThreshold = 5
   250  
   251  	for j, err := range errors {
   252  		if j == i || err == nil {
   253  			continue
   254  		}
   255  
   256  		if j >= errLogThreshold {
   257  			break
   258  		}
   259  
   260  		h.Log.Errorln(err)
   261  	}
   262  
   263  	return errors[i]
   264  }
   265  
   266  func (h *HugoSites) IsMultihost() bool {
   267  	return h != nil && h.multihost
   268  }
   269  
   270  // TODO(bep) consolidate
   271  func (h *HugoSites) LanguageSet() map[string]int {
   272  	set := make(map[string]int)
   273  	for i, s := range h.Sites {
   274  		set[s.language.Lang] = i
   275  	}
   276  	return set
   277  }
   278  
   279  func (h *HugoSites) NumLogErrors() int {
   280  	if h == nil {
   281  		return 0
   282  	}
   283  	return int(h.Log.LogCounters().ErrorCounter.Count())
   284  }
   285  
   286  func (h *HugoSites) PrintProcessingStats(w io.Writer) {
   287  	stats := make([]*helpers.ProcessingStats, len(h.Sites))
   288  	for i := 0; i < len(h.Sites); i++ {
   289  		stats[i] = h.Sites[i].PathSpec.ProcessingStats
   290  	}
   291  	helpers.ProcessingStatsTable(w, stats...)
   292  }
   293  
   294  // GetContentPage finds a Page with content given the absolute filename.
   295  // Returns nil if none found.
   296  func (h *HugoSites) GetContentPage(filename string) page.Page {
   297  	var p page.Page
   298  
   299  	h.getContentMaps().walkBundles(func(b *contentNode) bool {
   300  		if b.p == nil || b.fi == nil {
   301  			return false
   302  		}
   303  
   304  		if b.fi.Meta().Filename == filename {
   305  			p = b.p
   306  			return true
   307  		}
   308  
   309  		return false
   310  	})
   311  
   312  	return p
   313  }
   314  
   315  // NewHugoSites creates a new collection of sites given the input sites, building
   316  // a language configuration based on those.
   317  func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
   318  	if cfg.Language != nil {
   319  		return nil, errors.New("Cannot provide Language in Cfg when sites are provided")
   320  	}
   321  
   322  	// Return error at the end. Make the caller decide if it's fatal or not.
   323  	var initErr error
   324  
   325  	langConfig, err := newMultiLingualFromSites(cfg.Cfg, sites...)
   326  	if err != nil {
   327  		return nil, errors.Wrap(err, "failed to create language config")
   328  	}
   329  
   330  	var contentChangeTracker *contentChangeMap
   331  
   332  	numWorkers := config.GetNumWorkerMultiplier()
   333  	if numWorkers > len(sites) {
   334  		numWorkers = len(sites)
   335  	}
   336  	var workers *para.Workers
   337  	if numWorkers > 1 {
   338  		workers = para.New(numWorkers)
   339  	}
   340  
   341  	h := &HugoSites{
   342  		running:                 cfg.Running,
   343  		multilingual:            langConfig,
   344  		multihost:               cfg.Cfg.GetBool("multihost"),
   345  		Sites:                   sites,
   346  		workers:                 workers,
   347  		numWorkers:              numWorkers,
   348  		skipRebuildForFilenames: make(map[string]bool),
   349  		init: &hugoSitesInit{
   350  			data:         lazy.New(),
   351  			layouts:      lazy.New(),
   352  			gitInfo:      lazy.New(),
   353  			translations: lazy.New(),
   354  		},
   355  	}
   356  
   357  	h.fatalErrorHandler = &fatalErrorHandler{
   358  		h:     h,
   359  		donec: make(chan bool),
   360  	}
   361  
   362  	h.init.data.Add(func() (interface{}, error) {
   363  		err := h.loadData(h.PathSpec.BaseFs.Data.Dirs)
   364  		if err != nil {
   365  			return nil, errors.Wrap(err, "failed to load data")
   366  		}
   367  		return nil, nil
   368  	})
   369  
   370  	h.init.layouts.Add(func() (interface{}, error) {
   371  		for _, s := range h.Sites {
   372  			if err := s.Tmpl().(tpl.TemplateManager).MarkReady(); err != nil {
   373  				return nil, err
   374  			}
   375  		}
   376  		return nil, nil
   377  	})
   378  
   379  	h.init.translations.Add(func() (interface{}, error) {
   380  		if len(h.Sites) > 1 {
   381  			allTranslations := pagesToTranslationsMap(h.Sites)
   382  			assignTranslationsToPages(allTranslations, h.Sites)
   383  		}
   384  
   385  		return nil, nil
   386  	})
   387  
   388  	h.init.gitInfo.Add(func() (interface{}, error) {
   389  		err := h.loadGitInfo()
   390  		if err != nil {
   391  			return nil, errors.Wrap(err, "failed to load Git info")
   392  		}
   393  		return nil, nil
   394  	})
   395  
   396  	for _, s := range sites {
   397  		s.h = h
   398  	}
   399  
   400  	var l configLoader
   401  	if err := l.applyDeps(cfg, sites...); err != nil {
   402  		initErr = errors.Wrap(err, "add site dependencies")
   403  	}
   404  
   405  	h.Deps = sites[0].Deps
   406  	if h.Deps == nil {
   407  		return nil, initErr
   408  	}
   409  
   410  	// Only needed in server mode.
   411  	// TODO(bep) clean up the running vs watching terms
   412  	if cfg.Running {
   413  		contentChangeTracker = &contentChangeMap{
   414  			pathSpec:      h.PathSpec,
   415  			symContent:    make(map[string]map[string]bool),
   416  			leafBundles:   radix.New(),
   417  			branchBundles: make(map[string]bool),
   418  		}
   419  		h.ContentChanges = contentChangeTracker
   420  	}
   421  
   422  	return h, initErr
   423  }
   424  
   425  func (h *HugoSites) loadGitInfo() error {
   426  	if h.Cfg.GetBool("enableGitInfo") {
   427  		gi, err := newGitInfo(h.Cfg)
   428  		if err != nil {
   429  			h.Log.Errorln("Failed to read Git log:", err)
   430  		} else {
   431  			h.gitInfo = gi
   432  		}
   433  
   434  		co, err := newCodeowners(h.Cfg)
   435  		if err != nil {
   436  			h.Log.Errorln("Failed to read CODEOWNERS:", err)
   437  		} else {
   438  			h.codeownerInfo = co
   439  		}
   440  	}
   441  	return nil
   442  }
   443  
   444  func (l configLoader) applyDeps(cfg deps.DepsCfg, sites ...*Site) error {
   445  	if cfg.TemplateProvider == nil {
   446  		cfg.TemplateProvider = tplimpl.DefaultTemplateProvider
   447  	}
   448  
   449  	if cfg.TranslationProvider == nil {
   450  		cfg.TranslationProvider = i18n.NewTranslationProvider()
   451  	}
   452  
   453  	var (
   454  		d   *deps.Deps
   455  		err error
   456  	)
   457  
   458  	for _, s := range sites {
   459  		if s.Deps != nil {
   460  			continue
   461  		}
   462  
   463  		onCreated := func(d *deps.Deps) error {
   464  			s.Deps = d
   465  
   466  			// Set up the main publishing chain.
   467  			pub, err := publisher.NewDestinationPublisher(
   468  				d.ResourceSpec,
   469  				s.outputFormatsConfig,
   470  				s.mediaTypesConfig,
   471  			)
   472  			if err != nil {
   473  				return err
   474  			}
   475  			s.publisher = pub
   476  
   477  			if err := s.initializeSiteInfo(); err != nil {
   478  				return err
   479  			}
   480  
   481  			d.Site = s.Info
   482  
   483  			siteConfig, err := l.loadSiteConfig(s.language)
   484  			if err != nil {
   485  				return errors.Wrap(err, "load site config")
   486  			}
   487  			s.siteConfigConfig = siteConfig
   488  
   489  			pm := &pageMap{
   490  				contentMap: newContentMap(contentMapConfig{
   491  					lang:                 s.Lang(),
   492  					taxonomyConfig:       s.siteCfg.taxonomiesConfig.Values(),
   493  					taxonomyDisabled:     !s.isEnabled(page.KindTerm),
   494  					taxonomyTermDisabled: !s.isEnabled(page.KindTaxonomy),
   495  					pageDisabled:         !s.isEnabled(page.KindPage),
   496  				}),
   497  				s: s,
   498  			}
   499  
   500  			s.PageCollections = newPageCollections(pm)
   501  
   502  			s.siteRefLinker, err = newSiteRefLinker(s.language, s)
   503  			return err
   504  		}
   505  
   506  		cfg.Language = s.language
   507  		cfg.MediaTypes = s.mediaTypesConfig
   508  		cfg.OutputFormats = s.outputFormatsConfig
   509  
   510  		if d == nil {
   511  			cfg.WithTemplate = s.withSiteTemplates(cfg.WithTemplate)
   512  
   513  			var err error
   514  			d, err = deps.New(cfg)
   515  			if err != nil {
   516  				return errors.Wrap(err, "create deps")
   517  			}
   518  
   519  			d.OutputFormatsConfig = s.outputFormatsConfig
   520  
   521  			if err := onCreated(d); err != nil {
   522  				return errors.Wrap(err, "on created")
   523  			}
   524  
   525  			if err = d.LoadResources(); err != nil {
   526  				return errors.Wrap(err, "load resources")
   527  			}
   528  
   529  		} else {
   530  			d, err = d.ForLanguage(cfg, onCreated)
   531  			if err != nil {
   532  				return err
   533  			}
   534  			d.OutputFormatsConfig = s.outputFormatsConfig
   535  		}
   536  	}
   537  
   538  	return nil
   539  }
   540  
   541  // NewHugoSites creates HugoSites from the given config.
   542  func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
   543  	if cfg.Logger == nil {
   544  		cfg.Logger = loggers.NewErrorLogger()
   545  	}
   546  	sites, err := createSitesFromConfig(cfg)
   547  	if err != nil {
   548  		return nil, errors.Wrap(err, "from config")
   549  	}
   550  	return newHugoSites(cfg, sites...)
   551  }
   552  
   553  func (s *Site) withSiteTemplates(withTemplates ...func(templ tpl.TemplateManager) error) func(templ tpl.TemplateManager) error {
   554  	return func(templ tpl.TemplateManager) error {
   555  		for _, wt := range withTemplates {
   556  			if wt == nil {
   557  				continue
   558  			}
   559  			if err := wt(templ); err != nil {
   560  				return err
   561  			}
   562  		}
   563  
   564  		return nil
   565  	}
   566  }
   567  
   568  func createSitesFromConfig(cfg deps.DepsCfg) ([]*Site, error) {
   569  	var sites []*Site
   570  
   571  	languages := getLanguages(cfg.Cfg)
   572  
   573  	for _, lang := range languages {
   574  		if lang.Disabled {
   575  			continue
   576  		}
   577  		var s *Site
   578  		var err error
   579  		cfg.Language = lang
   580  		s, err = newSite(cfg)
   581  
   582  		if err != nil {
   583  			return nil, err
   584  		}
   585  
   586  		sites = append(sites, s)
   587  	}
   588  
   589  	return sites, nil
   590  }
   591  
   592  // Reset resets the sites and template caches etc., making it ready for a full rebuild.
   593  func (h *HugoSites) reset(config *BuildCfg) {
   594  	if config.ResetState {
   595  		for i, s := range h.Sites {
   596  			h.Sites[i] = s.reset()
   597  			if r, ok := s.Fs.Destination.(hugofs.Reseter); ok {
   598  				r.Reset()
   599  			}
   600  		}
   601  	}
   602  
   603  	h.fatalErrorHandler = &fatalErrorHandler{
   604  		h:     h,
   605  		donec: make(chan bool),
   606  	}
   607  
   608  	h.init.Reset()
   609  }
   610  
   611  // resetLogs resets the log counters etc. Used to do a new build on the same sites.
   612  func (h *HugoSites) resetLogs() {
   613  	h.Log.Reset()
   614  	loggers.GlobalErrorCounter.Reset()
   615  	for _, s := range h.Sites {
   616  		s.Deps.Log.Reset()
   617  		s.Deps.LogDistinct.Reset()
   618  	}
   619  }
   620  
   621  func (h *HugoSites) withSite(fn func(s *Site) error) error {
   622  	if h.workers == nil {
   623  		for _, s := range h.Sites {
   624  			if err := fn(s); err != nil {
   625  				return err
   626  			}
   627  		}
   628  		return nil
   629  	}
   630  
   631  	g, _ := h.workers.Start(context.Background())
   632  	for _, s := range h.Sites {
   633  		s := s
   634  		g.Run(func() error {
   635  			return fn(s)
   636  		})
   637  	}
   638  	return g.Wait()
   639  }
   640  
   641  func (h *HugoSites) createSitesFromConfig(cfg config.Provider) error {
   642  	oldLangs, _ := h.Cfg.Get("languagesSorted").(langs.Languages)
   643  
   644  	l := configLoader{cfg: h.Cfg}
   645  	if err := l.loadLanguageSettings(oldLangs); err != nil {
   646  		return err
   647  	}
   648  
   649  	depsCfg := deps.DepsCfg{Fs: h.Fs, Cfg: l.cfg}
   650  
   651  	sites, err := createSitesFromConfig(depsCfg)
   652  	if err != nil {
   653  		return err
   654  	}
   655  
   656  	langConfig, err := newMultiLingualFromSites(depsCfg.Cfg, sites...)
   657  	if err != nil {
   658  		return err
   659  	}
   660  
   661  	h.Sites = sites
   662  
   663  	for _, s := range sites {
   664  		s.h = h
   665  	}
   666  
   667  	var cl configLoader
   668  	if err := cl.applyDeps(depsCfg, sites...); err != nil {
   669  		return err
   670  	}
   671  
   672  	h.Deps = sites[0].Deps
   673  
   674  	h.multilingual = langConfig
   675  	h.multihost = h.Deps.Cfg.GetBool("multihost")
   676  
   677  	return nil
   678  }
   679  
   680  func (h *HugoSites) toSiteInfos() []*SiteInfo {
   681  	infos := make([]*SiteInfo, len(h.Sites))
   682  	for i, s := range h.Sites {
   683  		infos[i] = s.Info
   684  	}
   685  	return infos
   686  }
   687  
   688  // BuildCfg holds build options used to, as an example, skip the render step.
   689  type BuildCfg struct {
   690  	// Reset site state before build. Use to force full rebuilds.
   691  	ResetState bool
   692  	// If set, we re-create the sites from the given configuration before a build.
   693  	// This is needed if new languages are added.
   694  	NewConfig config.Provider
   695  	// Skip rendering. Useful for testing.
   696  	SkipRender bool
   697  	// Use this to indicate what changed (for rebuilds).
   698  	whatChanged *whatChanged
   699  
   700  	// This is a partial re-render of some selected pages. This means
   701  	// we should skip most of the processing.
   702  	PartialReRender bool
   703  
   704  	// Set in server mode when the last build failed for some reason.
   705  	ErrRecovery bool
   706  
   707  	// Recently visited URLs. This is used for partial re-rendering.
   708  	RecentlyVisited map[string]bool
   709  
   710  	// Can be set to build only with a sub set of the content source.
   711  	ContentInclusionFilter *glob.FilenameFilter
   712  
   713  	// Set when the buildlock is already acquired (e.g. the archetype content builder).
   714  	NoBuildLock bool
   715  
   716  	testCounters *testCounters
   717  }
   718  
   719  // shouldRender is used in the Fast Render Mode to determine if we need to re-render
   720  // a Page: If it is recently visited (the home pages will always be in this set) or changed.
   721  // Note that a page does not have to have a content page / file.
   722  // For regular builds, this will allways return true.
   723  // TODO(bep) rename/work this.
   724  func (cfg *BuildCfg) shouldRender(p *pageState) bool {
   725  	if p.forceRender {
   726  		return true
   727  	}
   728  
   729  	if len(cfg.RecentlyVisited) == 0 {
   730  		return true
   731  	}
   732  
   733  	if cfg.RecentlyVisited[p.RelPermalink()] {
   734  		return true
   735  	}
   736  
   737  	if cfg.whatChanged != nil && !p.File().IsZero() {
   738  		return cfg.whatChanged.files[p.File().Filename()]
   739  	}
   740  
   741  	return false
   742  }
   743  
   744  func (h *HugoSites) renderCrossSitesSitemap() error {
   745  	if !h.multilingual.enabled() || h.IsMultihost() {
   746  		return nil
   747  	}
   748  
   749  	sitemapEnabled := false
   750  	for _, s := range h.Sites {
   751  		if s.isEnabled(kindSitemap) {
   752  			sitemapEnabled = true
   753  			break
   754  		}
   755  	}
   756  
   757  	if !sitemapEnabled {
   758  		return nil
   759  	}
   760  
   761  	s := h.Sites[0]
   762  
   763  	templ := s.lookupLayouts("sitemapindex.xml", "_default/sitemapindex.xml", "_internal/_default/sitemapindex.xml")
   764  
   765  	return s.renderAndWriteXML(&s.PathSpec.ProcessingStats.Sitemaps, "sitemapindex",
   766  		s.siteCfg.sitemap.Filename, h.toSiteInfos(), templ)
   767  }
   768  
   769  func (h *HugoSites) renderCrossSitesRobotsTXT() error {
   770  	if h.multihost {
   771  		return nil
   772  	}
   773  	if !h.Cfg.GetBool("enableRobotsTXT") {
   774  		return nil
   775  	}
   776  
   777  	s := h.Sites[0]
   778  
   779  	p, err := newPageStandalone(&pageMeta{
   780  		s:    s,
   781  		kind: kindRobotsTXT,
   782  		urlPaths: pagemeta.URLPath{
   783  			URL: "robots.txt",
   784  		},
   785  	},
   786  		output.RobotsTxtFormat)
   787  	if err != nil {
   788  		return err
   789  	}
   790  
   791  	if !p.render {
   792  		return nil
   793  	}
   794  
   795  	templ := s.lookupLayouts("robots.txt", "_default/robots.txt", "_internal/_default/robots.txt")
   796  
   797  	return s.renderAndWritePage(&s.PathSpec.ProcessingStats.Pages, "Robots Txt", "robots.txt", p, templ)
   798  }
   799  
   800  func (h *HugoSites) removePageByFilename(filename string) {
   801  	h.getContentMaps().withMaps(func(m *pageMap) error {
   802  		m.deleteBundleMatching(func(b *contentNode) bool {
   803  			if b.p == nil {
   804  				return false
   805  			}
   806  
   807  			if b.fi == nil {
   808  				return false
   809  			}
   810  
   811  			return b.fi.Meta().Filename == filename
   812  		})
   813  		return nil
   814  	})
   815  }
   816  
   817  func (h *HugoSites) createPageCollections() error {
   818  	allPages := newLazyPagesFactory(func() page.Pages {
   819  		var pages page.Pages
   820  		for _, s := range h.Sites {
   821  			pages = append(pages, s.Pages()...)
   822  		}
   823  
   824  		page.SortByDefault(pages)
   825  
   826  		return pages
   827  	})
   828  
   829  	allRegularPages := newLazyPagesFactory(func() page.Pages {
   830  		return h.findPagesByKindIn(page.KindPage, allPages.get())
   831  	})
   832  
   833  	for _, s := range h.Sites {
   834  		s.PageCollections.allPages = allPages
   835  		s.PageCollections.allRegularPages = allRegularPages
   836  	}
   837  
   838  	return nil
   839  }
   840  
   841  func (s *Site) preparePagesForRender(isRenderingSite bool, idx int) error {
   842  	var err error
   843  	s.pageMap.withEveryBundlePage(func(p *pageState) bool {
   844  		if err = p.initOutputFormat(isRenderingSite, idx); err != nil {
   845  			return true
   846  		}
   847  		return false
   848  	})
   849  	return nil
   850  }
   851  
   852  // Pages returns all pages for all sites.
   853  func (h *HugoSites) Pages() page.Pages {
   854  	return h.Sites[0].AllPages()
   855  }
   856  
   857  func (h *HugoSites) loadData(fis []hugofs.FileMetaInfo) (err error) {
   858  	spec := source.NewSourceSpec(h.PathSpec, nil, nil)
   859  
   860  	h.data = make(map[string]interface{})
   861  	for _, fi := range fis {
   862  		fileSystem := spec.NewFilesystemFromFileMetaInfo(fi)
   863  		files, err := fileSystem.Files()
   864  		if err != nil {
   865  			return err
   866  		}
   867  		for _, r := range files {
   868  			if err := h.handleDataFile(r); err != nil {
   869  				return err
   870  			}
   871  		}
   872  	}
   873  
   874  	return
   875  }
   876  
   877  func (h *HugoSites) handleDataFile(r source.File) error {
   878  	var current map[string]interface{}
   879  
   880  	f, err := r.FileInfo().Meta().Open()
   881  	if err != nil {
   882  		return errors.Wrapf(err, "data: failed to open %q:", r.LogicalName())
   883  	}
   884  	defer f.Close()
   885  
   886  	// Crawl in data tree to insert data
   887  	current = h.data
   888  	keyParts := strings.Split(r.Dir(), helpers.FilePathSeparator)
   889  
   890  	for _, key := range keyParts {
   891  		if key != "" {
   892  			if _, ok := current[key]; !ok {
   893  				current[key] = make(map[string]interface{})
   894  			}
   895  			current = current[key].(map[string]interface{})
   896  		}
   897  	}
   898  
   899  	data, err := h.readData(r)
   900  	if err != nil {
   901  		return h.errWithFileContext(err, r)
   902  	}
   903  
   904  	if data == nil {
   905  		return nil
   906  	}
   907  
   908  	// filepath.Walk walks the files in lexical order, '/' comes before '.'
   909  	higherPrecedentData := current[r.BaseFileName()]
   910  
   911  	switch data.(type) {
   912  	case nil:
   913  	case map[string]interface{}:
   914  
   915  		switch higherPrecedentData.(type) {
   916  		case nil:
   917  			current[r.BaseFileName()] = data
   918  		case map[string]interface{}:
   919  			// merge maps: insert entries from data for keys that
   920  			// don't already exist in higherPrecedentData
   921  			higherPrecedentMap := higherPrecedentData.(map[string]interface{})
   922  			for key, value := range data.(map[string]interface{}) {
   923  				if _, exists := higherPrecedentMap[key]; exists {
   924  					// this warning could happen if
   925  					// 1. A theme uses the same key; the main data folder wins
   926  					// 2. A sub folder uses the same key: the sub folder wins
   927  					// TODO(bep) figure out a way to detect 2) above and make that a WARN
   928  					h.Log.Infof("Data for key '%s' in path '%s' is overridden by higher precedence data already in the data tree", key, r.Path())
   929  				} else {
   930  					higherPrecedentMap[key] = value
   931  				}
   932  			}
   933  		default:
   934  			// can't merge: higherPrecedentData is not a map
   935  			h.Log.Warnf("The %T data from '%s' overridden by "+
   936  				"higher precedence %T data already in the data tree", data, r.Path(), higherPrecedentData)
   937  		}
   938  
   939  	case []interface{}:
   940  		if higherPrecedentData == nil {
   941  			current[r.BaseFileName()] = data
   942  		} else {
   943  			// we don't merge array data
   944  			h.Log.Warnf("The %T data from '%s' overridden by "+
   945  				"higher precedence %T data already in the data tree", data, r.Path(), higherPrecedentData)
   946  		}
   947  
   948  	default:
   949  		h.Log.Errorf("unexpected data type %T in file %s", data, r.LogicalName())
   950  	}
   951  
   952  	return nil
   953  }
   954  
   955  func (h *HugoSites) errWithFileContext(err error, f source.File) error {
   956  	fim, ok := f.FileInfo().(hugofs.FileMetaInfo)
   957  	if !ok {
   958  		return err
   959  	}
   960  
   961  	realFilename := fim.Meta().Filename
   962  
   963  	err, _ = herrors.WithFileContextForFile(
   964  		err,
   965  		realFilename,
   966  		realFilename,
   967  		h.SourceSpec.Fs.Source,
   968  		herrors.SimpleLineMatcher)
   969  
   970  	return err
   971  }
   972  
   973  func (h *HugoSites) readData(f source.File) (interface{}, error) {
   974  	file, err := f.FileInfo().Meta().Open()
   975  	if err != nil {
   976  		return nil, errors.Wrap(err, "readData: failed to open data file")
   977  	}
   978  	defer file.Close()
   979  	content := helpers.ReaderToBytes(file)
   980  
   981  	format := metadecoders.FormatFromString(f.Extension())
   982  	return metadecoders.Default.Unmarshal(content, format)
   983  }
   984  
   985  func (h *HugoSites) findPagesByKindIn(kind string, inPages page.Pages) page.Pages {
   986  	return h.Sites[0].findPagesByKindIn(kind, inPages)
   987  }
   988  
   989  func (h *HugoSites) resetPageState() {
   990  	h.getContentMaps().walkBundles(func(n *contentNode) bool {
   991  		if n.p == nil {
   992  			return false
   993  		}
   994  		p := n.p
   995  		for _, po := range p.pageOutputs {
   996  			if po.cp == nil {
   997  				continue
   998  			}
   999  			po.cp.Reset()
  1000  		}
  1001  
  1002  		return false
  1003  	})
  1004  }
  1005  
  1006  func (h *HugoSites) resetPageStateFromEvents(idset identity.Identities) {
  1007  	h.getContentMaps().walkBundles(func(n *contentNode) bool {
  1008  		if n.p == nil {
  1009  			return false
  1010  		}
  1011  		p := n.p
  1012  	OUTPUTS:
  1013  		for _, po := range p.pageOutputs {
  1014  			if po.cp == nil {
  1015  				continue
  1016  			}
  1017  			for id := range idset {
  1018  				if po.cp.dependencyTracker.Search(id) != nil {
  1019  					po.cp.Reset()
  1020  					continue OUTPUTS
  1021  				}
  1022  			}
  1023  		}
  1024  
  1025  		if p.shortcodeState == nil {
  1026  			return false
  1027  		}
  1028  
  1029  		for _, s := range p.shortcodeState.shortcodes {
  1030  			for _, templ := range s.templs {
  1031  				sid := templ.(identity.Manager)
  1032  				for id := range idset {
  1033  					if sid.Search(id) != nil {
  1034  						for _, po := range p.pageOutputs {
  1035  							if po.cp != nil {
  1036  								po.cp.Reset()
  1037  							}
  1038  						}
  1039  						return false
  1040  					}
  1041  				}
  1042  			}
  1043  		}
  1044  		return false
  1045  	})
  1046  }
  1047  
  1048  // Used in partial reloading to determine if the change is in a bundle.
  1049  type contentChangeMap struct {
  1050  	mu sync.RWMutex
  1051  
  1052  	// Holds directories with leaf bundles.
  1053  	leafBundles *radix.Tree
  1054  
  1055  	// Holds directories with branch bundles.
  1056  	branchBundles map[string]bool
  1057  
  1058  	pathSpec *helpers.PathSpec
  1059  
  1060  	// Hugo supports symlinked content (both directories and files). This
  1061  	// can lead to situations where the same file can be referenced from several
  1062  	// locations in /content -- which is really cool, but also means we have to
  1063  	// go an extra mile to handle changes.
  1064  	// This map is only used in watch mode.
  1065  	// It maps either file to files or the real dir to a set of content directories
  1066  	// where it is in use.
  1067  	symContentMu sync.Mutex
  1068  	symContent   map[string]map[string]bool
  1069  }
  1070  
  1071  func (m *contentChangeMap) add(dirname string, tp bundleDirType) {
  1072  	m.mu.Lock()
  1073  	if !strings.HasSuffix(dirname, helpers.FilePathSeparator) {
  1074  		dirname += helpers.FilePathSeparator
  1075  	}
  1076  	switch tp {
  1077  	case bundleBranch:
  1078  		m.branchBundles[dirname] = true
  1079  	case bundleLeaf:
  1080  		m.leafBundles.Insert(dirname, true)
  1081  	default:
  1082  		m.mu.Unlock()
  1083  		panic("invalid bundle type")
  1084  	}
  1085  	m.mu.Unlock()
  1086  }
  1087  
  1088  func (m *contentChangeMap) resolveAndRemove(filename string) (string, bundleDirType) {
  1089  	m.mu.RLock()
  1090  	defer m.mu.RUnlock()
  1091  
  1092  	// Bundles share resources, so we need to start from the virtual root.
  1093  	relFilename := m.pathSpec.RelContentDir(filename)
  1094  	dir, name := filepath.Split(relFilename)
  1095  	if !strings.HasSuffix(dir, helpers.FilePathSeparator) {
  1096  		dir += helpers.FilePathSeparator
  1097  	}
  1098  
  1099  	if _, found := m.branchBundles[dir]; found {
  1100  		delete(m.branchBundles, dir)
  1101  		return dir, bundleBranch
  1102  	}
  1103  
  1104  	if key, _, found := m.leafBundles.LongestPrefix(dir); found {
  1105  		m.leafBundles.Delete(key)
  1106  		dir = string(key)
  1107  		return dir, bundleLeaf
  1108  	}
  1109  
  1110  	fileTp, isContent := classifyBundledFile(name)
  1111  	if isContent && fileTp != bundleNot {
  1112  		// A new bundle.
  1113  		return dir, fileTp
  1114  	}
  1115  
  1116  	return dir, bundleNot
  1117  }
  1118  
  1119  func (m *contentChangeMap) addSymbolicLinkMapping(fim hugofs.FileMetaInfo) {
  1120  	meta := fim.Meta()
  1121  	if !meta.IsSymlink {
  1122  		return
  1123  	}
  1124  	m.symContentMu.Lock()
  1125  
  1126  	from, to := meta.Filename, meta.OriginalFilename
  1127  	if fim.IsDir() {
  1128  		if !strings.HasSuffix(from, helpers.FilePathSeparator) {
  1129  			from += helpers.FilePathSeparator
  1130  		}
  1131  	}
  1132  
  1133  	mm, found := m.symContent[from]
  1134  
  1135  	if !found {
  1136  		mm = make(map[string]bool)
  1137  		m.symContent[from] = mm
  1138  	}
  1139  	mm[to] = true
  1140  	m.symContentMu.Unlock()
  1141  }
  1142  
  1143  func (m *contentChangeMap) GetSymbolicLinkMappings(dir string) []string {
  1144  	mm, found := m.symContent[dir]
  1145  	if !found {
  1146  		return nil
  1147  	}
  1148  	dirs := make([]string, len(mm))
  1149  	i := 0
  1150  	for dir := range mm {
  1151  		dirs[i] = dir
  1152  		i++
  1153  	}
  1154  
  1155  	sort.Strings(dirs)
  1156  
  1157  	return dirs
  1158  }