github.com/neohugo/neohugo@v0.123.8/hugolib/content_map_page.go (about) 1 // Copyright 2024 The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package hugolib 15 16 import ( 17 "context" 18 "fmt" 19 "io" 20 "path" 21 "sort" 22 "strconv" 23 "strings" 24 "sync" 25 "sync/atomic" 26 "time" 27 28 "github.com/bep/logg" 29 "github.com/neohugo/neohugo/cache/dynacache" 30 "github.com/neohugo/neohugo/common/loggers" 31 "github.com/neohugo/neohugo/common/paths" 32 "github.com/neohugo/neohugo/common/predicate" 33 "github.com/neohugo/neohugo/common/rungroup" 34 "github.com/neohugo/neohugo/common/types" 35 "github.com/neohugo/neohugo/hugofs/files" 36 "github.com/neohugo/neohugo/hugolib/doctree" 37 "github.com/neohugo/neohugo/identity" 38 "github.com/neohugo/neohugo/output" 39 "github.com/neohugo/neohugo/resources" 40 "github.com/spf13/cast" 41 42 "github.com/neohugo/neohugo/common/maps" 43 44 "github.com/neohugo/neohugo/resources/kinds" 45 "github.com/neohugo/neohugo/resources/page" 46 "github.com/neohugo/neohugo/resources/page/pagemeta" 47 "github.com/neohugo/neohugo/resources/resource" 48 ) 49 50 var pagePredicates = struct { 51 KindPage predicate.P[*pageState] 52 KindSection predicate.P[*pageState] 53 KindHome predicate.P[*pageState] 54 KindTerm predicate.P[*pageState] 55 ShouldListLocal predicate.P[*pageState] 56 ShouldListGlobal predicate.P[*pageState] 57 ShouldListAny predicate.P[*pageState] 58 ShouldLink predicate.P[page.Page] 59 }{ 60 KindPage: func(p *pageState) bool { 61 return p.Kind() == kinds.KindPage 62 }, 63 KindSection: func(p *pageState) bool { 64 return p.Kind() == kinds.KindSection 65 }, 66 KindHome: func(p *pageState) bool { 67 return p.Kind() == kinds.KindHome 68 }, 69 KindTerm: func(p *pageState) bool { 70 return p.Kind() == kinds.KindTerm 71 }, 72 ShouldListLocal: func(p *pageState) bool { 73 return p.m.shouldList(false) 74 }, 75 ShouldListGlobal: func(p *pageState) bool { 76 return p.m.shouldList(true) 77 }, 78 ShouldListAny: func(p *pageState) bool { 79 return p.m.shouldListAny() 80 }, 81 ShouldLink: func(p page.Page) bool { 82 return !p.(*pageState).m.noLink() 83 }, 84 } 85 86 type pageMap struct { 87 i int 88 s *Site 89 90 // Main storage for all pages. 91 *pageTrees 92 93 // Used for simple page lookups by name, e.g. "mypage.md" or "mypage". 94 pageReverseIndex *contentTreeReverseIndex 95 96 cachePages *dynacache.Partition[string, page.Pages] 97 cacheResources *dynacache.Partition[string, resource.Resources] 98 cacheContentRendered *dynacache.Partition[string, *resources.StaleValue[contentSummary]] 99 cacheContentPlain *dynacache.Partition[string, *resources.StaleValue[contentPlainPlainWords]] 100 contentTableOfContents *dynacache.Partition[string, *resources.StaleValue[contentTableOfContents]] 101 102 cfg contentMapConfig 103 } 104 105 // pageTrees holds pages and resources in a tree structure for all sites/languages. 106 // Each site gets its own tree set via the Shape method. 107 type pageTrees struct { 108 // This tree contains all Pages. 109 // This include regular pages, sections, taxonomies and so on. 110 // Note that all of these trees share the same key structure, 111 // so you can take a leaf Page key and do a prefix search 112 // with key + "/" to get all of its resources. 113 treePages *doctree.NodeShiftTree[contentNodeI] 114 115 // This tree contains Resources bundled in pages. 116 treeResources *doctree.NodeShiftTree[contentNodeI] 117 118 // All pages and resources. 119 treePagesResources doctree.WalkableTrees[contentNodeI] 120 121 // This tree contains all taxonomy entries, e.g "/tags/blue/page1" 122 treeTaxonomyEntries *doctree.TreeShiftTree[*weightedContentNode] 123 124 // A slice of the resource trees. 125 resourceTrees doctree.MutableTrees 126 } 127 128 // collectAndMarkStaleIdentities collects all identities from in all trees matching the given key. 129 // We currently re-read all page/resources for all languages that share the same path, 130 // so we mark all entries as stale (which will trigger cache invalidation), then 131 // return the first. 132 func (t *pageTrees) collectAndMarkStaleIdentities(p *paths.Path) []identity.Identity { 133 key := p.Base() 134 var ids []identity.Identity 135 // We need only one identity sample per dimensio. 136 nCount := 0 137 cb := func(n contentNodeI) bool { 138 if n == nil { 139 return false 140 } 141 n.MarkStale() 142 if nCount > 0 { 143 return true 144 } 145 nCount++ 146 n.ForEeachIdentity(func(id identity.Identity) bool { 147 ids = append(ids, id) 148 return false 149 }) 150 151 return false 152 } 153 tree := t.treePages 154 nCount = 0 155 tree.ForEeachInDimension(key, doctree.DimensionLanguage.Index(), 156 cb, 157 ) 158 159 tree = t.treeResources 160 nCount = 0 161 tree.ForEeachInDimension(key, doctree.DimensionLanguage.Index(), 162 cb, 163 ) 164 165 if p.Component() == files.ComponentFolderContent { 166 // It may also be a bundled content resource. 167 key := p.ForBundleType(paths.PathTypeContentResource).Base() 168 tree = t.treeResources 169 nCount = 0 170 tree.ForEeachInDimension(key, doctree.DimensionLanguage.Index(), 171 cb, 172 ) 173 174 } 175 return ids 176 } 177 178 // collectIdentitiesSurrounding collects all identities surrounding the given key. 179 func (t *pageTrees) collectIdentitiesSurrounding(key string, maxSamplesPerTree int) []identity.Identity { 180 ids := t.collectIdentitiesSurroundingIn(key, maxSamplesPerTree, t.treePages) 181 ids = append(ids, t.collectIdentitiesSurroundingIn(key, maxSamplesPerTree, t.treeResources)...) 182 return ids 183 } 184 185 func (t *pageTrees) collectIdentitiesSurroundingIn(key string, maxSamples int, tree *doctree.NodeShiftTree[contentNodeI]) []identity.Identity { 186 var ids []identity.Identity 187 section, ok := tree.LongestPrefixAll(path.Dir(key)) 188 if ok { 189 count := 0 190 prefix := section + "/" 191 level := strings.Count(prefix, "/") 192 tree.WalkPrefixRaw(prefix, func(s string, n contentNodeI) bool { 193 if level != strings.Count(s, "/") { 194 return false 195 } 196 n.ForEeachIdentity(func(id identity.Identity) bool { 197 ids = append(ids, id) 198 return false 199 }) 200 count++ 201 return count > maxSamples 202 }) 203 } 204 205 return ids 206 } 207 208 func (t *pageTrees) DeletePageAndResourcesBelow(ss ...string) { 209 commit1 := t.resourceTrees.Lock(true) 210 defer commit1() 211 commit2 := t.treePages.Lock(true) 212 defer commit2() 213 for _, s := range ss { 214 t.resourceTrees.DeletePrefix(paths.AddTrailingSlash(s)) 215 t.treePages.Delete(s) 216 } 217 } 218 219 // Shape shapes all trees in t to the given dimension. 220 func (t pageTrees) Shape(d, v int) *pageTrees { 221 t.treePages = t.treePages.Shape(d, v) 222 t.treeResources = t.treeResources.Shape(d, v) 223 t.treeTaxonomyEntries = t.treeTaxonomyEntries.Shape(d, v) 224 t.createMutableTrees() 225 226 return &t 227 } 228 229 func (t *pageTrees) createMutableTrees() { 230 t.treePagesResources = doctree.WalkableTrees[contentNodeI]{ 231 t.treePages, 232 t.treeResources, 233 } 234 235 t.resourceTrees = doctree.MutableTrees{ 236 t.treeResources, 237 } 238 } 239 240 var ( 241 _ resource.Identifier = pageMapQueryPagesInSection{} 242 _ resource.Identifier = pageMapQueryPagesBelowPath{} 243 ) 244 245 type pageMapQueryPagesInSection struct { 246 pageMapQueryPagesBelowPath 247 248 Recursive bool 249 IncludeSelf bool 250 } 251 252 func (q pageMapQueryPagesInSection) Key() string { 253 return "gagesInSection" + "/" + q.pageMapQueryPagesBelowPath.Key() + "/" + strconv.FormatBool(q.Recursive) + "/" + strconv.FormatBool(q.IncludeSelf) 254 } 255 256 // This needs to be hashable. 257 type pageMapQueryPagesBelowPath struct { 258 Path string 259 260 // Additional identifier for this query. 261 // Used as part of the cache key. 262 KeyPart string 263 264 // Page inclusion filter. 265 // May be nil. 266 Include predicate.P[*pageState] 267 } 268 269 func (q pageMapQueryPagesBelowPath) Key() string { 270 return q.Path + "/" + q.KeyPart 271 } 272 273 // Apply fn to all pages in m matching the given predicate. 274 // fn may return true to stop the walk. 275 func (m *pageMap) forEachPage(include predicate.P[*pageState], fn func(p *pageState) (bool, error)) error { 276 if include == nil { 277 include = func(p *pageState) bool { 278 return true 279 } 280 } 281 w := &doctree.NodeShiftTreeWalker[contentNodeI]{ 282 Tree: m.treePages, 283 LockType: doctree.LockTypeRead, 284 Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { 285 if p, ok := n.(*pageState); ok && include(p) { 286 if terminate, err := fn(p); terminate || err != nil { 287 return terminate, err 288 } 289 } 290 return false, nil 291 }, 292 } 293 294 return w.Walk(context.Background()) 295 } 296 297 func (m *pageMap) forEeachPageIncludingBundledPages(include predicate.P[*pageState], fn func(p *pageState) (bool, error)) error { 298 if include == nil { 299 include = func(p *pageState) bool { 300 return true 301 } 302 } 303 304 if err := m.forEachPage(include, fn); err != nil { 305 return err 306 } 307 308 w := &doctree.NodeShiftTreeWalker[contentNodeI]{ 309 Tree: m.treeResources, 310 LockType: doctree.LockTypeRead, 311 Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { 312 if rs, ok := n.(*resourceSource); ok { 313 if p, ok := rs.r.(*pageState); ok && include(p) { 314 if terminate, err := fn(p); terminate || err != nil { 315 return terminate, err 316 } 317 } 318 } 319 return false, nil 320 }, 321 } 322 323 return w.Walk(context.Background()) 324 } 325 326 func (m *pageMap) getOrCreatePagesFromCache( 327 key string, create func(string) (page.Pages, error), 328 ) (page.Pages, error) { 329 return m.cachePages.GetOrCreate(key, create) 330 } 331 332 func (m *pageMap) getPagesInSection(q pageMapQueryPagesInSection) page.Pages { 333 cacheKey := q.Key() 334 335 pages, err := m.getOrCreatePagesFromCache(cacheKey, func(string) (page.Pages, error) { 336 prefix := paths.AddTrailingSlash(q.Path) 337 338 var ( 339 pas page.Pages 340 otherBranch string 341 ) 342 343 include := q.Include 344 if include == nil { 345 include = pagePredicates.ShouldListLocal 346 } 347 348 w := &doctree.NodeShiftTreeWalker[contentNodeI]{ 349 Tree: m.treePages, 350 Prefix: prefix, 351 } 352 353 w.Handle = func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { 354 if q.Recursive { 355 if p, ok := n.(*pageState); ok && include(p) { 356 pas = append(pas, p) 357 } 358 return false, nil 359 } 360 361 if p, ok := n.(*pageState); ok && include(p) { 362 pas = append(pas, p) 363 } 364 365 if n.isContentNodeBranch() { 366 currentBranch := key + "/" 367 if otherBranch == "" || otherBranch != currentBranch { 368 w.SkipPrefix(currentBranch) 369 } 370 otherBranch = currentBranch 371 } 372 return false, nil 373 } 374 375 err := w.Walk(context.Background()) 376 377 if err == nil { 378 if q.IncludeSelf { 379 if n := m.treePages.Get(q.Path); n != nil { 380 if p, ok := n.(*pageState); ok && include(p) { 381 pas = append(pas, p) 382 } 383 } 384 } 385 page.SortByDefault(pas) 386 } 387 388 return pas, err 389 }) 390 if err != nil { 391 panic(err) 392 } 393 394 return pages 395 } 396 397 func (m *pageMap) getPagesWithTerm(q pageMapQueryPagesBelowPath) page.Pages { 398 key := q.Key() 399 400 v, err := m.cachePages.GetOrCreate(key, func(string) (page.Pages, error) { 401 var pas page.Pages 402 include := q.Include 403 if include == nil { 404 include = pagePredicates.ShouldListLocal 405 } 406 407 err := m.treeTaxonomyEntries.WalkPrefix( 408 doctree.LockTypeNone, 409 paths.AddTrailingSlash(q.Path), 410 func(s string, n *weightedContentNode) (bool, error) { 411 p := n.n.(*pageState) 412 if !include(p) { 413 return false, nil 414 } 415 pas = append(pas, pageWithWeight0{n.weight, p}) 416 return false, nil 417 }, 418 ) 419 if err != nil { 420 return nil, err 421 } 422 423 page.SortByDefault(pas) 424 425 return pas, nil 426 }) 427 if err != nil { 428 panic(err) 429 } 430 431 return v 432 } 433 434 func (m *pageMap) getTermsForPageInTaxonomy(path, taxonomy string) page.Pages { 435 prefix := paths.AddLeadingSlash(taxonomy) 436 437 v, err := m.cachePages.GetOrCreate(prefix+path, func(string) (page.Pages, error) { 438 var pas page.Pages 439 440 err := m.treeTaxonomyEntries.WalkPrefix( 441 doctree.LockTypeNone, 442 paths.AddTrailingSlash(prefix), 443 func(s string, n *weightedContentNode) (bool, error) { 444 if strings.HasSuffix(s, path) { 445 pas = append(pas, n.term) 446 } 447 return false, nil 448 }, 449 ) 450 if err != nil { 451 return nil, err 452 } 453 454 page.SortByDefault(pas) 455 456 return pas, nil 457 }) 458 if err != nil { 459 panic(err) 460 } 461 462 return v 463 } 464 465 func (m *pageMap) forEachResourceInPage( 466 ps *pageState, 467 lockType doctree.LockType, 468 exact bool, 469 handle func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error), 470 ) error { 471 keyPage := ps.Path() 472 if keyPage == "/" { 473 keyPage = "" 474 } 475 prefix := paths.AddTrailingSlash(ps.Path()) 476 isBranch := ps.IsNode() 477 478 rw := &doctree.NodeShiftTreeWalker[contentNodeI]{ 479 Tree: m.treeResources, 480 Prefix: prefix, 481 LockType: lockType, 482 Exact: exact, 483 } 484 485 rw.Handle = func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { 486 if isBranch { 487 ownerKey, _ := m.treePages.LongestPrefixAll(resourceKey) 488 if ownerKey != keyPage && path.Dir(ownerKey) != path.Dir(resourceKey) { 489 // Stop walking downwards, someone else owns this resource. 490 rw.SkipPrefix(ownerKey + "/") 491 return false, nil 492 } 493 } 494 return handle(resourceKey, n, match) 495 } 496 497 return rw.Walk(context.Background()) 498 } 499 500 func (m *pageMap) getResourcesForPage(ps *pageState) (resource.Resources, error) { 501 var res resource.Resources 502 // nolint 503 m.forEachResourceInPage(ps, doctree.LockTypeNone, false, func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { 504 rs := n.(*resourceSource) 505 if rs.r != nil { 506 res = append(res, rs.r) 507 } 508 return false, nil 509 }) 510 return res, nil 511 } 512 513 func (m *pageMap) getOrCreateResourcesForPage(ps *pageState) resource.Resources { 514 keyPage := ps.Path() 515 if keyPage == "/" { 516 keyPage = "" 517 } 518 key := keyPage + "/get-resources-for-page" 519 v, err := m.cacheResources.GetOrCreate(key, func(string) (resource.Resources, error) { 520 res, err := m.getResourcesForPage(ps) 521 if err != nil { 522 return nil, err 523 } 524 525 if translationKey := ps.m.pageConfig.TranslationKey; translationKey != "" { 526 // This this should not be a very common case. 527 // Merge in resources from the other languages. 528 translatedPages, _ := m.s.h.translationKeyPages.Get(translationKey) 529 for _, tp := range translatedPages { 530 if tp == ps { 531 continue 532 } 533 tps := tp.(*pageState) 534 // Make sure we query from the correct language root. 535 res2, err := tps.s.pageMap.getResourcesForPage(tps) 536 if err != nil { 537 return nil, err 538 } 539 // Add if Name not already in res. 540 for _, r := range res2 { 541 var found bool 542 for _, r2 := range res { 543 if r2.(resource.NameNormalizedProvider).NameNormalized() == r.(resource.NameNormalizedProvider).NameNormalized() { 544 found = true 545 break 546 } 547 } 548 if !found { 549 res = append(res, r) 550 } 551 } 552 } 553 } 554 555 lessFunc := func(i, j int) bool { 556 ri, rj := res[i], res[j] 557 if ri.ResourceType() < rj.ResourceType() { 558 return true 559 } 560 561 p1, ok1 := ri.(page.Page) 562 p2, ok2 := rj.(page.Page) 563 564 if ok1 != ok2 { 565 // Pull pages behind other resources. 566 567 return ok2 568 } 569 570 if ok1 { 571 return page.DefaultPageSort(p1, p2) 572 } 573 574 // Make sure not to use RelPermalink or any of the other methods that 575 // trigger lazy publishing. 576 return ri.Name() < rj.Name() 577 } 578 579 sort.SliceStable(res, lessFunc) 580 581 if len(ps.m.pageConfig.Resources) > 0 { 582 for i, r := range res { 583 res[i] = resources.CloneWithMetadataIfNeeded(ps.m.pageConfig.Resources, r) 584 } 585 sort.SliceStable(res, lessFunc) 586 } 587 588 return res, nil 589 }) 590 if err != nil { 591 panic(err) 592 } 593 594 return v 595 } 596 597 type weightedContentNode struct { 598 n contentNodeI 599 weight int 600 term *pageWithOrdinal 601 } 602 603 type buildStateReseter interface { 604 resetBuildState() 605 } 606 607 type contentNodeI interface { 608 identity.IdentityProvider 609 identity.ForEeachIdentityProvider 610 Path() string 611 isContentNodeBranch() bool 612 buildStateReseter 613 resource.StaleMarker 614 } 615 616 var _ contentNodeI = (*contentNodeIs)(nil) 617 618 type contentNodeIs []contentNodeI 619 620 func (n contentNodeIs) Path() string { 621 return n[0].Path() 622 } 623 624 func (n contentNodeIs) isContentNodeBranch() bool { 625 return n[0].isContentNodeBranch() 626 } 627 628 func (n contentNodeIs) GetIdentity() identity.Identity { 629 return n[0].GetIdentity() 630 } 631 632 func (n contentNodeIs) ForEeachIdentity(f func(identity.Identity) bool) bool { 633 for _, nn := range n { 634 if nn != nil { 635 if nn.ForEeachIdentity(f) { 636 return true 637 } 638 } 639 } 640 return false 641 } 642 643 func (n contentNodeIs) resetBuildState() { 644 for _, nn := range n { 645 if nn != nil { 646 nn.resetBuildState() 647 } 648 } 649 } 650 651 func (n contentNodeIs) MarkStale() { 652 for _, nn := range n { 653 resource.MarkStale(nn) 654 } 655 } 656 657 type contentNodeShifter struct { 658 numLanguages int 659 } 660 661 func (s *contentNodeShifter) Delete(n contentNodeI, dimension doctree.Dimension) (bool, bool) { 662 lidx := dimension[0] 663 switch v := n.(type) { 664 case contentNodeIs: 665 resource.MarkStale(v[lidx]) 666 wasDeleted := v[lidx] != nil 667 v[lidx] = nil 668 isEmpty := true 669 for _, vv := range v { 670 if vv != nil { 671 isEmpty = false 672 break 673 } 674 } 675 return wasDeleted, isEmpty 676 case resourceSources: 677 resource.MarkStale(v[lidx]) 678 wasDeleted := v[lidx] != nil 679 v[lidx] = nil 680 isEmpty := true 681 for _, vv := range v { 682 if vv != nil { 683 isEmpty = false 684 break 685 } 686 } 687 return wasDeleted, isEmpty 688 case *resourceSource: 689 if lidx != v.LangIndex() { 690 return false, false 691 } 692 resource.MarkStale(v) 693 return true, true 694 case *pageState: 695 if lidx != v.s.languagei { 696 return false, false 697 } 698 resource.MarkStale(v) 699 return true, true 700 default: 701 panic(fmt.Sprintf("unknown type %T", n)) 702 } 703 } 704 705 func (s *contentNodeShifter) Shift(n contentNodeI, dimension doctree.Dimension, exact bool) (contentNodeI, bool, doctree.DimensionFlag) { 706 lidx := dimension[0] 707 // How accurate is the match. 708 accuracy := doctree.DimensionLanguage 709 switch v := n.(type) { 710 case contentNodeIs: 711 if len(v) == 0 { 712 panic("empty contentNodeIs") 713 } 714 vv := v[lidx] 715 if vv != nil { 716 return vv, true, accuracy 717 } 718 return nil, false, 0 719 case resourceSources: 720 vv := v[lidx] 721 if vv != nil { 722 return vv, true, doctree.DimensionLanguage 723 } 724 if exact { 725 return nil, false, 0 726 } 727 // For non content resources, pick the first match. 728 for _, vv := range v { 729 if vv != nil { 730 if vv.isPage() { 731 return nil, false, 0 732 } 733 return vv, true, 0 734 } 735 } 736 case *resourceSource: 737 if v.LangIndex() == lidx { 738 return v, true, doctree.DimensionLanguage 739 } 740 if !v.isPage() && !exact { 741 return v, true, 0 742 } 743 case *pageState: 744 if v.s.languagei == lidx { 745 return n, true, doctree.DimensionLanguage 746 } 747 default: 748 panic(fmt.Sprintf("unknown type %T", n)) 749 } 750 return nil, false, 0 751 } 752 753 func (s *contentNodeShifter) ForEeachInDimension(n contentNodeI, d int, f func(contentNodeI) bool) { 754 if d != doctree.DimensionLanguage.Index() { 755 panic("only language dimension supported") 756 } 757 758 switch vv := n.(type) { 759 case contentNodeIs: 760 for _, v := range vv { 761 if v != nil { 762 if f(v) { 763 return 764 } 765 } 766 } 767 default: 768 f(vv) 769 } 770 } 771 772 func (s *contentNodeShifter) InsertInto(old, new contentNodeI, dimension doctree.Dimension) contentNodeI { 773 langi := dimension[doctree.DimensionLanguage.Index()] 774 switch vv := old.(type) { 775 case *pageState: 776 newp, ok := new.(*pageState) 777 if !ok { 778 panic(fmt.Sprintf("unknown type %T", new)) 779 } 780 if vv.s.languagei == newp.s.languagei && newp.s.languagei == langi { 781 return new 782 } 783 is := make(contentNodeIs, s.numLanguages) 784 is[vv.s.languagei] = old 785 is[langi] = new 786 return is 787 case contentNodeIs: 788 vv[langi] = new 789 return vv 790 case resourceSources: 791 vv[langi] = new.(*resourceSource) 792 return vv 793 case *resourceSource: 794 newp, ok := new.(*resourceSource) 795 if !ok { 796 panic(fmt.Sprintf("unknown type %T", new)) 797 } 798 if vv.LangIndex() == newp.LangIndex() && newp.LangIndex() == langi { 799 return new 800 } 801 rs := make(resourceSources, s.numLanguages) 802 rs[vv.LangIndex()] = vv 803 rs[langi] = newp 804 return rs 805 806 default: 807 panic(fmt.Sprintf("unknown type %T", old)) 808 } 809 } 810 811 func (s *contentNodeShifter) Insert(old, new contentNodeI) contentNodeI { 812 switch vv := old.(type) { 813 case *pageState: 814 newp, ok := new.(*pageState) 815 if !ok { 816 panic(fmt.Sprintf("unknown type %T", new)) 817 } 818 if vv.s.languagei == newp.s.languagei { 819 return new 820 } 821 is := make(contentNodeIs, s.numLanguages) 822 is[newp.s.languagei] = new 823 is[vv.s.languagei] = old 824 return is 825 case contentNodeIs: 826 newp, ok := new.(*pageState) 827 if !ok { 828 panic(fmt.Sprintf("unknown type %T", new)) 829 } 830 resource.MarkStale(vv[newp.s.languagei]) 831 vv[newp.s.languagei] = new 832 return vv 833 case *resourceSource: 834 newp, ok := new.(*resourceSource) 835 if !ok { 836 panic(fmt.Sprintf("unknown type %T", new)) 837 } 838 if vv.LangIndex() == newp.LangIndex() { 839 return new 840 } 841 rs := make(resourceSources, s.numLanguages) 842 rs[newp.LangIndex()] = newp 843 rs[vv.LangIndex()] = vv 844 return rs 845 case resourceSources: 846 newp, ok := new.(*resourceSource) 847 if !ok { 848 panic(fmt.Sprintf("unknown type %T", new)) 849 } 850 resource.MarkStale(vv[newp.LangIndex()]) 851 vv[newp.LangIndex()] = newp 852 return vv 853 default: 854 panic(fmt.Sprintf("unknown type %T", old)) 855 } 856 } 857 858 func newPageMap(i int, s *Site, mcache *dynacache.Cache, pageTrees *pageTrees) *pageMap { 859 var m *pageMap 860 861 var taxonomiesConfig taxonomiesConfig = s.conf.Taxonomies 862 863 m = &pageMap{ 864 pageTrees: pageTrees.Shape(0, i), 865 cachePages: dynacache.GetOrCreatePartition[string, page.Pages](mcache, fmt.Sprintf("/pags/%d", i), dynacache.OptionsPartition{Weight: 10, ClearWhen: dynacache.ClearOnRebuild}), 866 cacheResources: dynacache.GetOrCreatePartition[string, resource.Resources](mcache, fmt.Sprintf("/ress/%d", i), dynacache.OptionsPartition{Weight: 60, ClearWhen: dynacache.ClearOnRebuild}), 867 cacheContentRendered: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentSummary]](mcache, fmt.Sprintf("/cont/ren/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}), 868 cacheContentPlain: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentPlainPlainWords]](mcache, fmt.Sprintf("/cont/pla/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}), 869 contentTableOfContents: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentTableOfContents]](mcache, fmt.Sprintf("/cont/toc/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}), 870 871 cfg: contentMapConfig{ 872 lang: s.Lang(), 873 taxonomyConfig: taxonomiesConfig.Values(), 874 taxonomyDisabled: !s.conf.IsKindEnabled(kinds.KindTaxonomy), 875 taxonomyTermDisabled: !s.conf.IsKindEnabled(kinds.KindTerm), 876 pageDisabled: !s.conf.IsKindEnabled(kinds.KindPage), 877 }, 878 i: i, 879 s: s, 880 } 881 882 m.pageReverseIndex = &contentTreeReverseIndex{ 883 initFn: func(rm map[any]contentNodeI) { 884 add := func(k string, n contentNodeI) { 885 existing, found := rm[k] 886 if found && existing != ambiguousContentNode { 887 rm[k] = ambiguousContentNode 888 } else if !found { 889 rm[k] = n 890 } 891 } 892 893 w := &doctree.NodeShiftTreeWalker[contentNodeI]{ 894 Tree: m.treePages, 895 LockType: doctree.LockTypeRead, 896 Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { 897 p := n.(*pageState) 898 if p.File() != nil { 899 add(p.File().FileInfo().Meta().PathInfo.BaseNameNoIdentifier(), p) 900 } 901 return false, nil 902 }, 903 } 904 905 if err := w.Walk(context.Background()); err != nil { 906 panic(err) 907 } 908 }, 909 contentTreeReverseIndexMap: &contentTreeReverseIndexMap{}, 910 } 911 912 return m 913 } 914 915 type contentTreeReverseIndex struct { 916 initFn func(rm map[any]contentNodeI) 917 *contentTreeReverseIndexMap 918 } 919 920 func (c *contentTreeReverseIndex) Reset() { 921 c.contentTreeReverseIndexMap = &contentTreeReverseIndexMap{ 922 m: make(map[any]contentNodeI), 923 } 924 } 925 926 func (c *contentTreeReverseIndex) Get(key any) contentNodeI { 927 c.init.Do(func() { 928 c.m = make(map[any]contentNodeI) 929 c.initFn(c.contentTreeReverseIndexMap.m) 930 }) 931 return c.m[key] 932 } 933 934 type contentTreeReverseIndexMap struct { 935 init sync.Once 936 m map[any]contentNodeI 937 } 938 939 type sitePagesAssembler struct { 940 *Site 941 watching bool 942 incomingChanges *whatChanged 943 assembleChanges *whatChanged 944 ctx context.Context 945 } 946 947 func (m *pageMap) debugPrint(prefix string, maxLevel int, w io.Writer) { 948 noshift := false 949 var prevKey string 950 951 pageWalker := &doctree.NodeShiftTreeWalker[contentNodeI]{ 952 NoShift: noshift, 953 Tree: m.treePages, 954 Prefix: prefix, 955 WalkContext: &doctree.WalkContext[contentNodeI]{}, 956 } 957 958 resourceWalker := pageWalker.Extend() 959 resourceWalker.Tree = m.treeResources 960 961 pageWalker.Handle = func(keyPage string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { 962 level := strings.Count(keyPage, "/") 963 if level > maxLevel { 964 return false, nil 965 } 966 const indentStr = " " 967 p := n.(*pageState) 968 s := strings.TrimPrefix(keyPage, paths.CommonDir(prevKey, keyPage)) 969 lenIndent := len(keyPage) - len(s) 970 fmt.Fprint(w, strings.Repeat(indentStr, lenIndent)) 971 info := fmt.Sprintf("%s lm: %s (%s)", s, p.Lastmod().Format("2006-01-02"), p.Kind()) 972 fmt.Fprintln(w, info) 973 switch p.Kind() { 974 case kinds.KindTerm: 975 // nolint 976 m.treeTaxonomyEntries.WalkPrefix( 977 doctree.LockTypeNone, 978 keyPage+"/", 979 func(s string, n *weightedContentNode) (bool, error) { 980 fmt.Fprint(w, strings.Repeat(indentStr, lenIndent+4)) 981 fmt.Fprintln(w, s) 982 return false, nil 983 }, 984 ) 985 } 986 987 isBranch := n.isContentNodeBranch() 988 prevKey = keyPage 989 resourceWalker.Prefix = keyPage + "/" 990 991 resourceWalker.Handle = func(ss string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { 992 if isBranch { 993 ownerKey, _ := pageWalker.Tree.LongestPrefix(ss, true, nil) 994 if ownerKey != keyPage { 995 // Stop walking downwards, someone else owns this resource. 996 pageWalker.SkipPrefix(ownerKey + "/") 997 return false, nil 998 } 999 } 1000 fmt.Fprint(w, strings.Repeat(indentStr, lenIndent+8)) 1001 fmt.Fprintln(w, ss+" (resource)") 1002 return false, nil 1003 } 1004 1005 return false, resourceWalker.Walk(context.Background()) 1006 } 1007 1008 err := pageWalker.Walk(context.Background()) 1009 if err != nil { 1010 panic(err) 1011 } 1012 } 1013 1014 func (h *HugoSites) resolveAndClearStateForIdentities( 1015 ctx context.Context, 1016 l logg.LevelLogger, 1017 cachebuster func(s string) bool, changes []identity.Identity, 1018 ) error { 1019 h.Log.Debug().Log(logg.StringFunc( 1020 func() string { 1021 var sb strings.Builder 1022 for _, change := range changes { 1023 var key string 1024 if kp, ok := change.(resource.Identifier); ok { 1025 key = " " + kp.Key() 1026 } 1027 sb.WriteString(fmt.Sprintf("Direct dependencies of %q (%T%s) =>\n", change.IdentifierBase(), change, key)) 1028 seen := map[string]bool{ 1029 change.IdentifierBase(): true, 1030 } 1031 // Print the top level dependencies. 1032 identity.WalkIdentitiesDeep(change, func(level int, id identity.Identity) bool { 1033 if level > 1 { 1034 return true 1035 } 1036 if !seen[id.IdentifierBase()] { 1037 sb.WriteString(fmt.Sprintf(" %s%s\n", strings.Repeat(" ", level), id.IdentifierBase())) 1038 } 1039 seen[id.IdentifierBase()] = true 1040 return false 1041 }) 1042 } 1043 return sb.String() 1044 }), 1045 ) 1046 1047 for _, id := range changes { 1048 if staler, ok := id.(resource.Staler); ok && !staler.IsStale() { 1049 var msgDetail string 1050 if p, ok := id.(*pageState); ok && p.File() != nil { 1051 msgDetail = fmt.Sprintf(" (%s)", p.File().Filename()) 1052 } 1053 h.Log.Trace(logg.StringFunc(func() string { return fmt.Sprintf("Marking stale: %s (%T)%s\n", id, id, msgDetail) })) 1054 staler.MarkStale() 1055 } 1056 } 1057 1058 // The order matters here: 1059 // 1. Handle the cache busters first, as those may produce identities for the page reset step. 1060 // 2. Then reset the page outputs, which may mark some resources as stale. 1061 // 3. Then GC the cache. 1062 if cachebuster != nil { 1063 if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) { 1064 ll := l.WithField("substep", "gc dynacache cachebuster") 1065 1066 shouldDelete := func(k, v any) bool { 1067 if cachebuster == nil { 1068 return false 1069 } 1070 var b bool 1071 if s, ok := k.(string); ok { 1072 b = cachebuster(s) 1073 } 1074 1075 return b 1076 } 1077 1078 h.MemCache.ClearMatching(shouldDelete) 1079 1080 return ll, nil 1081 }); err != nil { 1082 return err 1083 } 1084 } 1085 1086 // Drain the the cache eviction stack. 1087 evicted := h.Deps.MemCache.DrainEvictedIdentities() 1088 if len(evicted) < 200 { 1089 changes = append(changes, evicted...) 1090 } else { 1091 // Mass eviction, we might as well invalidate everything. 1092 changes = []identity.Identity{identity.GenghisKhan} 1093 } 1094 1095 // Remove duplicates 1096 seen := make(map[identity.Identity]bool) 1097 var n int 1098 for _, id := range changes { 1099 if !seen[id] { 1100 seen[id] = true 1101 changes[n] = id 1102 n++ 1103 } 1104 } 1105 changes = changes[:n] 1106 1107 if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) { 1108 // changesLeft: The IDs that the pages is dependent on. 1109 // changesRight: The IDs that the pages depend on. 1110 ll := l.WithField("substep", "resolve page output change set").WithField("changes", len(changes)) 1111 1112 checkedCount, matchCount, err := h.resolveAndResetDependententPageOutputs(ctx, changes) 1113 ll = ll.WithField("checked", checkedCount).WithField("matches", matchCount) 1114 return ll, err 1115 }); err != nil { 1116 return err 1117 } 1118 1119 if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) { 1120 ll := l.WithField("substep", "gc dynacache") 1121 1122 h.MemCache.ClearOnRebuild(changes...) 1123 h.Log.Trace(logg.StringFunc(func() string { 1124 var sb strings.Builder 1125 sb.WriteString("dynacache keys:\n") 1126 for _, key := range h.MemCache.Keys(nil) { 1127 sb.WriteString(fmt.Sprintf(" %s\n", key)) 1128 } 1129 return sb.String() 1130 })) 1131 return ll, nil 1132 }); err != nil { 1133 return err 1134 } 1135 1136 return nil 1137 } 1138 1139 // The left change set is the IDs that the pages is dependent on. 1140 // The right change set is the IDs that the pages depend on. 1141 func (h *HugoSites) resolveAndResetDependententPageOutputs(ctx context.Context, changes []identity.Identity) (int, int, error) { 1142 if changes == nil { 1143 return 0, 0, nil 1144 } 1145 1146 // This can be shared (many of the same IDs are repeated). 1147 depsFinder := identity.NewFinder(identity.FinderConfig{}) 1148 1149 h.Log.Trace(logg.StringFunc(func() string { 1150 var sb strings.Builder 1151 sb.WriteString("resolve page dependencies: ") 1152 for _, id := range changes { 1153 sb.WriteString(fmt.Sprintf(" %T: %s|", id, id.IdentifierBase())) 1154 } 1155 return sb.String() 1156 })) 1157 1158 var ( 1159 resetCounter atomic.Int64 1160 checkedCounter atomic.Int64 1161 ) 1162 1163 resetPo := func(po *pageOutput, r identity.FinderResult) { 1164 if po.pco != nil { 1165 po.pco.Reset() // Will invalidate content cache. 1166 } 1167 1168 po.renderState = 0 1169 po.p.resourcesPublishInit = &sync.Once{} 1170 if r == identity.FinderFoundOneOfMany { 1171 // Will force a re-render even in fast render mode. 1172 po.renderOnce = false 1173 } 1174 resetCounter.Add(1) 1175 h.Log.Trace(logg.StringFunc(func() string { 1176 p := po.p 1177 return fmt.Sprintf("Resetting page output %s for %s for output %s\n", p.Kind(), p.Path(), po.f.Name) 1178 })) 1179 } 1180 1181 // This can be a relativeley expensive operations, so we do it in parallel. 1182 g := rungroup.Run[*pageState](ctx, rungroup.Config[*pageState]{ 1183 NumWorkers: h.numWorkers, 1184 Handle: func(ctx context.Context, p *pageState) error { 1185 if !p.isRenderedAny() { 1186 // This needs no reset, so no need to check it. 1187 return nil 1188 } 1189 // First check the top level dependency manager. 1190 for _, id := range changes { 1191 checkedCounter.Add(1) 1192 if r := depsFinder.Contains(id, p.dependencyManager, 2); r > identity.FinderFoundOneOfManyRepetition { 1193 for _, po := range p.pageOutputs { 1194 resetPo(po, r) 1195 } 1196 // Done. 1197 return nil 1198 } 1199 } 1200 // Then do a more fine grained reset for each output format. 1201 OUTPUTS: 1202 for _, po := range p.pageOutputs { 1203 if !po.isRendered() { 1204 continue 1205 } 1206 for _, id := range changes { 1207 checkedCounter.Add(1) 1208 if r := depsFinder.Contains(id, po.dependencyManagerOutput, 50); r > identity.FinderFoundOneOfManyRepetition { 1209 resetPo(po, r) 1210 continue OUTPUTS 1211 } 1212 } 1213 } 1214 return nil 1215 }, 1216 }) 1217 1218 h.withPage(func(s string, p *pageState) bool { 1219 var needToCheck bool 1220 for _, po := range p.pageOutputs { 1221 if po.isRendered() { 1222 needToCheck = true 1223 break 1224 } 1225 } 1226 if needToCheck { 1227 g.Enqueue(p) // nolint 1228 } 1229 return false 1230 }) 1231 1232 err := g.Wait() 1233 resetCount := int(resetCounter.Load()) 1234 checkedCount := int(checkedCounter.Load()) 1235 1236 return checkedCount, resetCount, err 1237 } 1238 1239 // Calculate and apply aggregate values to the page tree (e.g. dates, cascades). 1240 func (sa *sitePagesAssembler) applyAggregates() error { 1241 sectionPageCount := map[string]int{} 1242 1243 pw := &doctree.NodeShiftTreeWalker[contentNodeI]{ 1244 Tree: sa.pageMap.treePages, 1245 LockType: doctree.LockTypeRead, 1246 WalkContext: &doctree.WalkContext[contentNodeI]{}, 1247 } 1248 rw := pw.Extend() 1249 rw.Tree = sa.pageMap.treeResources 1250 sa.lastmod = time.Time{} 1251 1252 pw.Handle = func(keyPage string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { 1253 pageBundle := n.(*pageState) 1254 1255 if pageBundle.Kind() == kinds.KindTerm { 1256 // Delay this until they're created. 1257 return false, nil 1258 } 1259 1260 if pageBundle.IsPage() { 1261 rootSection := pageBundle.Section() 1262 sectionPageCount[rootSection]++ 1263 } 1264 1265 // Handle cascades first to get any default dates set. 1266 var cascade map[page.PageMatcher]maps.Params 1267 if keyPage == "" { 1268 // Home page gets it's cascade from the site config. 1269 cascade = sa.conf.Cascade.Config 1270 1271 if pageBundle.m.pageConfig.Cascade == nil { 1272 // Pass the site cascade downwards. 1273 pw.WalkContext.Data().Insert(keyPage, cascade) 1274 } 1275 } else { 1276 _, data := pw.WalkContext.Data().LongestPrefix(keyPage) 1277 if data != nil { 1278 cascade = data.(map[page.PageMatcher]maps.Params) 1279 } 1280 } 1281 1282 if (pageBundle.IsHome() || pageBundle.IsSection()) && pageBundle.m.setMetaPostCount > 0 { 1283 oldDates := pageBundle.m.pageConfig.Dates 1284 1285 // We need to wait until after the walk to determine if any of the dates have changed. 1286 pw.WalkContext.AddPostHook( 1287 func() error { 1288 if oldDates != pageBundle.m.pageConfig.Dates { 1289 sa.assembleChanges.Add(pageBundle) 1290 } 1291 return nil 1292 }, 1293 ) 1294 } 1295 1296 // Combine the cascade map with front matter. 1297 if err := pageBundle.setMetaPost(cascade); err != nil { 1298 return false, err 1299 } 1300 1301 // We receive cascade values from above. If this leads to a change compared 1302 // to the previous value, we need to mark the page and its dependencies as changed. 1303 if pageBundle.m.setMetaPostCascadeChanged { 1304 sa.assembleChanges.Add(pageBundle) 1305 } 1306 1307 const eventName = "dates" 1308 if n.isContentNodeBranch() { 1309 if pageBundle.m.pageConfig.Cascade != nil { 1310 // Pass it down. 1311 pw.WalkContext.Data().Insert(keyPage, pageBundle.m.pageConfig.Cascade) 1312 } 1313 1314 wasZeroDates := pageBundle.m.pageConfig.Dates.IsAllDatesZero() 1315 if wasZeroDates || pageBundle.IsHome() { 1316 pw.WalkContext.AddEventListener(eventName, keyPage, func(e *doctree.Event[contentNodeI]) { 1317 sp, ok := e.Source.(*pageState) 1318 if !ok { 1319 return 1320 } 1321 1322 if wasZeroDates { 1323 pageBundle.m.pageConfig.Dates.UpdateDateAndLastmodIfAfter(sp.m.pageConfig.Dates) 1324 } 1325 1326 if pageBundle.IsHome() { 1327 if pageBundle.m.pageConfig.Dates.Lastmod.After(pageBundle.s.lastmod) { 1328 pageBundle.s.lastmod = pageBundle.m.pageConfig.Dates.Lastmod 1329 } 1330 if sp.m.pageConfig.Dates.Lastmod.After(pageBundle.s.lastmod) { 1331 pageBundle.s.lastmod = sp.m.pageConfig.Dates.Lastmod 1332 } 1333 } 1334 }) 1335 } 1336 } 1337 1338 // Send the date info up the tree. 1339 pw.WalkContext.SendEvent(&doctree.Event[contentNodeI]{Source: n, Path: keyPage, Name: eventName}) 1340 1341 isBranch := n.isContentNodeBranch() 1342 rw.Prefix = keyPage + "/" 1343 1344 rw.Handle = func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { 1345 if isBranch { 1346 ownerKey, _ := pw.Tree.LongestPrefix(resourceKey, true, nil) 1347 if ownerKey != keyPage { 1348 // Stop walking downwards, someone else owns this resource. 1349 rw.SkipPrefix(ownerKey + "/") 1350 return false, nil 1351 } 1352 } 1353 rs := n.(*resourceSource) 1354 if rs.isPage() { 1355 pageResource := rs.r.(*pageState) 1356 relPath := pageResource.m.pathInfo.BaseRel(pageBundle.m.pathInfo) 1357 pageResource.m.resourcePath = relPath 1358 var cascade map[page.PageMatcher]maps.Params 1359 // Apply cascade (if set) to the page. 1360 _, data := pw.WalkContext.Data().LongestPrefix(resourceKey) 1361 if data != nil { 1362 cascade = data.(map[page.PageMatcher]maps.Params) 1363 } 1364 if err := pageResource.setMetaPost(cascade); err != nil { 1365 return false, err 1366 } 1367 } 1368 1369 return false, nil 1370 } 1371 return false, rw.Walk(sa.ctx) 1372 } 1373 1374 if err := pw.Walk(sa.ctx); err != nil { 1375 return err 1376 } 1377 1378 if err := pw.WalkContext.HandleEventsAndHooks(); err != nil { 1379 return err 1380 } 1381 1382 if !sa.s.conf.C.IsMainSectionsSet() { 1383 var mainSection string 1384 var maxcount int 1385 for section, counter := range sectionPageCount { 1386 if section != "" && counter > maxcount { 1387 mainSection = section 1388 maxcount = counter 1389 } 1390 } 1391 sa.s.conf.C.SetMainSections([]string{mainSection}) 1392 1393 } 1394 1395 return nil 1396 } 1397 1398 func (sa *sitePagesAssembler) applyAggregatesToTaxonomiesAndTerms() error { 1399 walkContext := &doctree.WalkContext[contentNodeI]{} 1400 1401 handlePlural := func(key string) error { 1402 var pw *doctree.NodeShiftTreeWalker[contentNodeI] 1403 pw = &doctree.NodeShiftTreeWalker[contentNodeI]{ 1404 Tree: sa.pageMap.treePages, 1405 Prefix: key, // We also want to include the root taxonomy nodes, so no trailing slash. 1406 LockType: doctree.LockTypeRead, 1407 WalkContext: walkContext, 1408 Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { 1409 p := n.(*pageState) 1410 if p.Kind() != kinds.KindTerm { 1411 // The other kinds were handled in applyAggregates. 1412 if p.m.pageConfig.Cascade != nil { 1413 // Pass it down. 1414 pw.WalkContext.Data().Insert(s, p.m.pageConfig.Cascade) 1415 } 1416 } 1417 1418 if p.Kind() != kinds.KindTerm && p.Kind() != kinds.KindTaxonomy { 1419 // Already handled. 1420 return false, nil 1421 } 1422 1423 const eventName = "dates" 1424 1425 if p.Kind() == kinds.KindTerm { 1426 var cascade map[page.PageMatcher]maps.Params 1427 _, data := pw.WalkContext.Data().LongestPrefix(s) 1428 if data != nil { 1429 cascade = data.(map[page.PageMatcher]maps.Params) 1430 } 1431 if err := p.setMetaPost(cascade); err != nil { 1432 return false, err 1433 } 1434 if !p.s.shouldBuild(p) { 1435 sa.pageMap.treePages.Delete(s) 1436 sa.pageMap.treeTaxonomyEntries.DeletePrefix(paths.AddTrailingSlash(s)) 1437 } else if err := sa.pageMap.treeTaxonomyEntries.WalkPrefix( 1438 doctree.LockTypeRead, 1439 paths.AddTrailingSlash(s), 1440 func(ss string, wn *weightedContentNode) (bool, error) { 1441 // Send the date info up the tree. 1442 pw.WalkContext.SendEvent(&doctree.Event[contentNodeI]{Source: wn.n, Path: ss, Name: eventName}) 1443 return false, nil 1444 }, 1445 ); err != nil { 1446 return false, err 1447 } 1448 } 1449 1450 // Send the date info up the tree. 1451 pw.WalkContext.SendEvent(&doctree.Event[contentNodeI]{Source: n, Path: s, Name: eventName}) 1452 1453 if p.m.pageConfig.Dates.IsAllDatesZero() { 1454 pw.WalkContext.AddEventListener(eventName, s, func(e *doctree.Event[contentNodeI]) { 1455 sp, ok := e.Source.(*pageState) 1456 if !ok { 1457 return 1458 } 1459 1460 p.m.pageConfig.Dates.UpdateDateAndLastmodIfAfter(sp.m.pageConfig.Dates) 1461 }) 1462 } 1463 1464 return false, nil 1465 }, 1466 } 1467 1468 if err := pw.Walk(sa.ctx); err != nil { 1469 return err 1470 } 1471 return nil 1472 } 1473 1474 for _, viewName := range sa.pageMap.cfg.taxonomyConfig.views { 1475 if err := handlePlural(viewName.pluralTreeKey); err != nil { 1476 return err 1477 } 1478 } 1479 1480 if err := walkContext.HandleEventsAndHooks(); err != nil { 1481 return err 1482 } 1483 1484 return nil 1485 } 1486 1487 func (sa *sitePagesAssembler) assembleTermsAndTranslations() error { 1488 var ( 1489 pages = sa.pageMap.treePages 1490 entries = sa.pageMap.treeTaxonomyEntries 1491 views = sa.pageMap.cfg.taxonomyConfig.views 1492 ) 1493 1494 lockType := doctree.LockTypeWrite 1495 w := &doctree.NodeShiftTreeWalker[contentNodeI]{ 1496 Tree: pages, 1497 LockType: lockType, 1498 Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { 1499 ps := n.(*pageState) 1500 1501 if ps.m.noLink() { 1502 return false, nil 1503 } 1504 1505 // This is a little out of place, but is conveniently put here. 1506 // Check if translationKey is set by user. 1507 // This is to support the manual way of setting the translationKey in front matter. 1508 if ps.m.pageConfig.TranslationKey != "" { 1509 sa.s.h.translationKeyPages.Append(ps.m.pageConfig.TranslationKey, ps) 1510 } 1511 1512 if sa.pageMap.cfg.taxonomyTermDisabled { 1513 return false, nil 1514 } 1515 1516 for _, viewName := range views { 1517 vals := types.ToStringSlicePreserveString(getParam(ps, viewName.plural, false)) 1518 if vals == nil { 1519 continue 1520 } 1521 1522 w := getParamToLower(ps, viewName.plural+"_weight") 1523 weight, err := cast.ToIntE(w) 1524 if err != nil { 1525 sa.Log.Warnf("Unable to convert taxonomy weight %#v to int for %q", w, n.Path()) 1526 // weight will equal zero, so let the flow continue 1527 } 1528 1529 for i, v := range vals { 1530 if v == "" { 1531 continue 1532 } 1533 viewTermKey := "/" + viewName.plural + "/" + v 1534 pi := sa.Site.Conf.PathParser().Parse(files.ComponentFolderContent, viewTermKey+"/_index.md") 1535 term := pages.Get(pi.Base()) 1536 if term == nil { 1537 m := &pageMeta{ 1538 term: v, 1539 singular: viewName.singular, 1540 s: sa.Site, 1541 pathInfo: pi, 1542 pageMetaParams: pageMetaParams{ 1543 pageConfig: &pagemeta.PageConfig{ 1544 Kind: kinds.KindTerm, 1545 }, 1546 }, 1547 } 1548 n, pi, err := sa.h.newPage(m) 1549 if err != nil { 1550 return false, err 1551 } 1552 pages.InsertIntoValuesDimension(pi.Base(), n) 1553 term = pages.Get(pi.Base()) 1554 } else { 1555 m := term.(*pageState).m 1556 m.term = v 1557 m.singular = viewName.singular 1558 } 1559 1560 if s == "" { 1561 // Consider making this the real value. 1562 s = "/" 1563 } 1564 1565 key := pi.Base() + s 1566 1567 entries.Insert(key, &weightedContentNode{ 1568 weight: weight, 1569 n: n, 1570 term: &pageWithOrdinal{pageState: term.(*pageState), ordinal: i}, 1571 }) 1572 } 1573 } 1574 return false, nil 1575 }, 1576 } 1577 1578 return w.Walk(sa.ctx) 1579 } 1580 1581 func (sa *sitePagesAssembler) assembleResources() error { 1582 pagesTree := sa.pageMap.treePages 1583 resourcesTree := sa.pageMap.treeResources 1584 1585 lockType := doctree.LockTypeWrite 1586 w := &doctree.NodeShiftTreeWalker[contentNodeI]{ 1587 Tree: pagesTree, 1588 LockType: lockType, 1589 Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { 1590 ps := n.(*pageState) 1591 1592 // Prepare resources for this page. 1593 ps.shiftToOutputFormat(true, 0) // nolint 1594 targetPaths := ps.targetPaths() 1595 baseTarget := targetPaths.SubResourceBaseTarget 1596 duplicateResourceFiles := true 1597 if ps.s.ContentSpec.Converters.IsGoldmark(ps.m.pageConfig.Markup) { 1598 duplicateResourceFiles = ps.s.ContentSpec.Converters.GetMarkupConfig().Goldmark.DuplicateResourceFiles 1599 } 1600 1601 duplicateResourceFiles = duplicateResourceFiles || ps.s.Conf.IsMultihost() 1602 // nolint 1603 sa.pageMap.forEachResourceInPage( 1604 ps, lockType, 1605 !duplicateResourceFiles, 1606 func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { 1607 rs := n.(*resourceSource) 1608 if !match.Has(doctree.DimensionLanguage) { 1609 // We got an alternative language version. 1610 // Clone this and insert it into the tree. 1611 rs = rs.clone() 1612 resourcesTree.InsertIntoCurrentDimension(resourceKey, rs) 1613 } 1614 if rs.r != nil { 1615 return false, nil 1616 } 1617 1618 relPathOriginal := rs.path.Unnormalized().PathRel(ps.m.pathInfo.Unnormalized()) 1619 relPath := rs.path.BaseRel(ps.m.pathInfo) 1620 1621 var targetBasePaths []string 1622 if ps.s.Conf.IsMultihost() { 1623 baseTarget = targetPaths.SubResourceBaseLink 1624 // In multihost we need to publish to the lang sub folder. 1625 targetBasePaths = []string{ps.s.GetTargetLanguageBasePath()} // TODO(bep) we don't need this as a slice anymore. 1626 1627 } 1628 1629 rd := resources.ResourceSourceDescriptor{ 1630 OpenReadSeekCloser: rs.opener, 1631 Path: rs.path, 1632 GroupIdentity: rs.path, 1633 TargetPath: relPathOriginal, // Use the original path for the target path, so the links can be guessed. 1634 TargetBasePaths: targetBasePaths, 1635 BasePathRelPermalink: targetPaths.SubResourceBaseLink, 1636 BasePathTargetPath: baseTarget, 1637 NameNormalized: relPath, 1638 NameOriginal: relPathOriginal, 1639 LazyPublish: !ps.m.pageConfig.Build.PublishResources, 1640 } 1641 r, err := ps.m.s.ResourceSpec.NewResource(rd) 1642 if err != nil { 1643 return false, err 1644 } 1645 rs.r = r 1646 return false, nil 1647 }, 1648 ) 1649 1650 return false, nil 1651 }, 1652 } 1653 1654 return w.Walk(sa.ctx) 1655 } 1656 1657 func (sa *sitePagesAssembler) assemblePagesStep1(ctx context.Context) error { 1658 if err := sa.addMissingTaxonomies(); err != nil { 1659 return err 1660 } 1661 if err := sa.addMissingRootSections(); err != nil { 1662 return err 1663 } 1664 if err := sa.addStandalonePages(); err != nil { 1665 return err 1666 } 1667 if err := sa.applyAggregates(); err != nil { 1668 return err 1669 } 1670 return nil 1671 } 1672 1673 func (sa *sitePagesAssembler) assemblePagesStep2() error { 1674 if err := sa.removeShouldNotBuild(); err != nil { 1675 return err 1676 } 1677 if err := sa.assembleTermsAndTranslations(); err != nil { 1678 return err 1679 } 1680 if err := sa.applyAggregatesToTaxonomiesAndTerms(); err != nil { 1681 return err 1682 } 1683 if err := sa.assembleResources(); err != nil { 1684 return err 1685 } 1686 return nil 1687 } 1688 1689 // Remove any leftover node that we should not build for some reason (draft, expired, scheduled in the future). 1690 // Note that for the home and section kinds we just disable the nodes to preserve the structure. 1691 func (sa *sitePagesAssembler) removeShouldNotBuild() error { 1692 s := sa.Site 1693 var keys []string 1694 w := &doctree.NodeShiftTreeWalker[contentNodeI]{ 1695 LockType: doctree.LockTypeRead, 1696 Tree: sa.pageMap.treePages, 1697 Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { 1698 p := n.(*pageState) 1699 if !s.shouldBuild(p) { 1700 switch p.Kind() { 1701 case kinds.KindHome, kinds.KindSection, kinds.KindTaxonomy: 1702 // We need to keep these for the structure, but disable 1703 // them so they don't get listed/rendered. 1704 (&p.m.pageConfig.Build).Disable() 1705 default: 1706 keys = append(keys, key) 1707 } 1708 } 1709 return false, nil 1710 }, 1711 } 1712 if err := w.Walk(sa.ctx); err != nil { 1713 return err 1714 } 1715 1716 if len(keys) == 0 { 1717 return nil 1718 } 1719 1720 sa.pageMap.DeletePageAndResourcesBelow(keys...) 1721 1722 return nil 1723 } 1724 1725 // // Create the fixed output pages, e.g. sitemap.xml, if not already there. 1726 func (sa *sitePagesAssembler) addStandalonePages() error { 1727 s := sa.Site 1728 m := s.pageMap 1729 tree := m.treePages 1730 1731 commit := tree.Lock(true) 1732 defer commit() 1733 1734 addStandalone := func(key, kind string, f output.Format) { 1735 if !s.Conf.IsMultihost() { 1736 switch kind { 1737 case kinds.KindSitemapIndex, kinds.KindRobotsTXT: 1738 // Only one for all languages. 1739 if s.languagei != 0 { 1740 return 1741 } 1742 } 1743 } 1744 1745 if !sa.Site.conf.IsKindEnabled(kind) || tree.Has(key) { 1746 return 1747 } 1748 1749 m := &pageMeta{ 1750 s: s, 1751 pathInfo: s.Conf.PathParser().Parse(files.ComponentFolderContent, key+f.MediaType.FirstSuffix.FullSuffix), 1752 pageMetaParams: pageMetaParams{ 1753 pageConfig: &pagemeta.PageConfig{ 1754 Kind: kind, 1755 }, 1756 }, 1757 standaloneOutputFormat: f, 1758 } 1759 1760 p, _, _ := s.h.newPage(m) 1761 1762 tree.InsertIntoValuesDimension(key, p) 1763 } 1764 1765 addStandalone("/404", kinds.KindStatus404, output.HTTPStatusHTMLFormat) 1766 1767 if s.conf.EnableRobotsTXT { 1768 if m.i == 0 || s.Conf.IsMultihost() { 1769 addStandalone("/_robots", kinds.KindRobotsTXT, output.RobotsTxtFormat) 1770 } 1771 } 1772 1773 sitemapEnabled := false 1774 for _, s := range s.h.Sites { 1775 if s.conf.IsKindEnabled(kinds.KindSitemap) { 1776 sitemapEnabled = true 1777 break 1778 } 1779 } 1780 1781 if sitemapEnabled { 1782 addStandalone("/_sitemap", kinds.KindSitemap, output.SitemapFormat) 1783 skipSitemapIndex := s.Conf.IsMultihost() || !(s.Conf.DefaultContentLanguageInSubdir() || s.Conf.IsMultiLingual()) 1784 1785 if !skipSitemapIndex { 1786 addStandalone("/_sitemapindex", kinds.KindSitemapIndex, output.SitemapIndexFormat) 1787 } 1788 } 1789 1790 return nil 1791 } 1792 1793 func (sa *sitePagesAssembler) addMissingRootSections() error { 1794 var hasHome bool 1795 1796 // Add missing root sections. 1797 seen := map[string]bool{} 1798 var w *doctree.NodeShiftTreeWalker[contentNodeI] 1799 w = &doctree.NodeShiftTreeWalker[contentNodeI]{ 1800 LockType: doctree.LockTypeWrite, 1801 Tree: sa.pageMap.treePages, 1802 Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { 1803 if n == nil { 1804 panic("n is nil") 1805 } 1806 1807 ps := n.(*pageState) 1808 1809 if ps.Lang() != sa.Lang() { 1810 panic(fmt.Sprintf("lang mismatch: %q: %s != %s", s, ps.Lang(), sa.Lang())) 1811 } 1812 1813 if s == "" { 1814 hasHome = true 1815 sa.home = ps 1816 return false, nil 1817 } 1818 1819 switch ps.Kind() { 1820 case kinds.KindPage, kinds.KindSection: 1821 // OK 1822 default: 1823 // Skip taxonomy nodes etc. 1824 return false, nil 1825 } 1826 1827 p := ps.m.pathInfo 1828 section := p.Section() 1829 if section == "" || seen[section] { 1830 return false, nil 1831 } 1832 seen[section] = true 1833 1834 // Try to preserve the original casing if possible. 1835 sectionUnnormalized := p.Unnormalized().Section() 1836 pth := sa.s.Conf.PathParser().Parse(files.ComponentFolderContent, "/"+sectionUnnormalized+"/_index.md") 1837 nn := w.Tree.Get(pth.Base()) 1838 1839 if nn == nil { 1840 m := &pageMeta{ 1841 s: sa.Site, 1842 pathInfo: pth, 1843 } 1844 1845 ps, pth, err := sa.h.newPage(m) 1846 if err != nil { 1847 return false, err 1848 } 1849 w.Tree.InsertIntoValuesDimension(pth.Base(), ps) 1850 } 1851 1852 // /a/b, we don't need to walk deeper. 1853 if strings.Count(s, "/") > 1 { 1854 w.SkipPrefix(s + "/") 1855 } 1856 1857 return false, nil 1858 }, 1859 } 1860 1861 if err := w.Walk(sa.ctx); err != nil { 1862 return err 1863 } 1864 1865 if !hasHome { 1866 p := sa.Site.Conf.PathParser().Parse(files.ComponentFolderContent, "/_index.md") 1867 m := &pageMeta{ 1868 s: sa.Site, 1869 pathInfo: p, 1870 pageMetaParams: pageMetaParams{ 1871 pageConfig: &pagemeta.PageConfig{ 1872 Kind: kinds.KindHome, 1873 }, 1874 }, 1875 } 1876 n, p, err := sa.h.newPage(m) 1877 if err != nil { 1878 return err 1879 } 1880 w.Tree.InsertWithLock(p.Base(), n) 1881 sa.home = n 1882 } 1883 1884 return nil 1885 } 1886 1887 func (sa *sitePagesAssembler) addMissingTaxonomies() error { 1888 if sa.pageMap.cfg.taxonomyDisabled && sa.pageMap.cfg.taxonomyTermDisabled { 1889 return nil 1890 } 1891 1892 tree := sa.pageMap.treePages 1893 1894 commit := tree.Lock(true) 1895 defer commit() 1896 1897 for _, viewName := range sa.pageMap.cfg.taxonomyConfig.views { 1898 key := viewName.pluralTreeKey 1899 if v := tree.Get(key); v == nil { 1900 m := &pageMeta{ 1901 s: sa.Site, 1902 pathInfo: sa.Conf.PathParser().Parse(files.ComponentFolderContent, key+"/_index.md"), 1903 pageMetaParams: pageMetaParams{ 1904 pageConfig: &pagemeta.PageConfig{ 1905 Kind: kinds.KindTaxonomy, 1906 }, 1907 }, 1908 singular: viewName.singular, 1909 } 1910 p, _, _ := sa.h.newPage(m) 1911 tree.InsertIntoValuesDimension(key, p) 1912 } 1913 } 1914 1915 return nil 1916 } 1917 1918 func (m *pageMap) CreateSiteTaxonomies(ctx context.Context) error { 1919 m.s.taxonomies = make(page.TaxonomyList) 1920 1921 if m.cfg.taxonomyDisabled && m.cfg.taxonomyTermDisabled { 1922 return nil 1923 } 1924 1925 for _, viewName := range m.cfg.taxonomyConfig.views { 1926 key := viewName.pluralTreeKey 1927 m.s.taxonomies[viewName.plural] = make(page.Taxonomy) 1928 w := &doctree.NodeShiftTreeWalker[contentNodeI]{ 1929 Tree: m.treePages, 1930 Prefix: paths.AddTrailingSlash(key), 1931 LockType: doctree.LockTypeRead, 1932 Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) { 1933 p := n.(*pageState) 1934 1935 switch p.Kind() { 1936 case kinds.KindTerm: 1937 if !p.m.shouldList(true) { 1938 return false, nil 1939 } 1940 taxonomy := m.s.taxonomies[viewName.plural] 1941 if taxonomy == nil { 1942 return true, fmt.Errorf("missing taxonomy: %s", viewName.plural) 1943 } 1944 if p.m.term == "" { 1945 panic("term is empty") 1946 } 1947 k := strings.ToLower(p.m.term) 1948 1949 err := m.treeTaxonomyEntries.WalkPrefix( 1950 doctree.LockTypeRead, 1951 paths.AddTrailingSlash(s), 1952 func(ss string, wn *weightedContentNode) (bool, error) { 1953 taxonomy[k] = append(taxonomy[k], page.NewWeightedPage(wn.weight, wn.n.(page.Page), wn.term.Page())) 1954 return false, nil 1955 }, 1956 ) 1957 if err != nil { 1958 return true, err 1959 } 1960 1961 default: 1962 return false, nil 1963 } 1964 1965 return false, nil 1966 }, 1967 } 1968 1969 if err := w.Walk(ctx); err != nil { 1970 return err 1971 } 1972 } 1973 1974 for _, taxonomy := range m.s.taxonomies { 1975 for _, v := range taxonomy { 1976 v.Sort() 1977 } 1978 } 1979 1980 return nil 1981 } 1982 1983 type viewName struct { 1984 singular string // e.g. "category" 1985 plural string // e.g. "categories" 1986 pluralTreeKey string 1987 } 1988 1989 func (v viewName) IsZero() bool { 1990 return v.singular == "" 1991 }