github.com/jbramsden/hugo@v0.47.1/hugolib/page_bundler.go (about) 1 // Copyright 2017-present The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package hugolib 15 16 import ( 17 "fmt" 18 "math" 19 "runtime" 20 21 // Use this until errgroup gets ported to context 22 // See https://github.com/golang/go/issues/19781 23 "golang.org/x/net/context" 24 "golang.org/x/sync/errgroup" 25 ) 26 27 type siteContentProcessor struct { 28 site *Site 29 30 handleContent contentHandler 31 32 ctx context.Context 33 34 // The input file bundles. 35 fileBundlesChan chan *bundleDir 36 37 // The input file singles. 38 fileSinglesChan chan *fileInfo 39 40 // These assets should be just copied to destination. 41 fileAssetsChan chan []pathLangFile 42 43 numWorkers int 44 45 // The output Pages 46 pagesChan chan *Page 47 48 // Used for partial rebuilds (aka. live reload) 49 // Will signal replacement of pages in the site collection. 50 partialBuild bool 51 } 52 53 func (s *siteContentProcessor) processBundle(b *bundleDir) { 54 select { 55 case s.fileBundlesChan <- b: 56 case <-s.ctx.Done(): 57 } 58 } 59 60 func (s *siteContentProcessor) processSingle(fi *fileInfo) { 61 select { 62 case s.fileSinglesChan <- fi: 63 case <-s.ctx.Done(): 64 } 65 } 66 67 func (s *siteContentProcessor) processAssets(assets []pathLangFile) { 68 select { 69 case s.fileAssetsChan <- assets: 70 case <-s.ctx.Done(): 71 } 72 } 73 74 func newSiteContentProcessor(ctx context.Context, partialBuild bool, s *Site) *siteContentProcessor { 75 numWorkers := 12 76 if n := runtime.NumCPU() * 3; n > numWorkers { 77 numWorkers = n 78 } 79 80 numWorkers = int(math.Ceil(float64(numWorkers) / float64(len(s.owner.Sites)))) 81 82 return &siteContentProcessor{ 83 ctx: ctx, 84 partialBuild: partialBuild, 85 site: s, 86 handleContent: newHandlerChain(s), 87 fileBundlesChan: make(chan *bundleDir, numWorkers), 88 fileSinglesChan: make(chan *fileInfo, numWorkers), 89 fileAssetsChan: make(chan []pathLangFile, numWorkers), 90 numWorkers: numWorkers, 91 pagesChan: make(chan *Page, numWorkers), 92 } 93 } 94 95 func (s *siteContentProcessor) closeInput() { 96 close(s.fileSinglesChan) 97 close(s.fileBundlesChan) 98 close(s.fileAssetsChan) 99 } 100 101 func (s *siteContentProcessor) process(ctx context.Context) error { 102 g1, ctx := errgroup.WithContext(ctx) 103 g2, ctx := errgroup.WithContext(ctx) 104 105 // There can be only one of these per site. 106 g1.Go(func() error { 107 for p := range s.pagesChan { 108 if p.s != s.site { 109 panic(fmt.Sprintf("invalid page site: %v vs %v", p.s, s)) 110 } 111 112 if s.partialBuild { 113 p.forceRender = true 114 s.site.replacePage(p) 115 } else { 116 s.site.addPage(p) 117 } 118 } 119 return nil 120 }) 121 122 for i := 0; i < s.numWorkers; i++ { 123 g2.Go(func() error { 124 for { 125 select { 126 case f, ok := <-s.fileSinglesChan: 127 if !ok { 128 return nil 129 } 130 err := s.readAndConvertContentFile(f) 131 if err != nil { 132 return err 133 } 134 case <-ctx.Done(): 135 return ctx.Err() 136 } 137 } 138 }) 139 140 g2.Go(func() error { 141 for { 142 select { 143 case files, ok := <-s.fileAssetsChan: 144 if !ok { 145 return nil 146 } 147 for _, file := range files { 148 f, err := s.site.BaseFs.Content.Fs.Open(file.Filename()) 149 if err != nil { 150 return fmt.Errorf("failed to open assets file: %s", err) 151 } 152 err = s.site.publish(&s.site.PathSpec.ProcessingStats.Files, file.Path(), f) 153 f.Close() 154 if err != nil { 155 return err 156 } 157 } 158 159 case <-ctx.Done(): 160 return ctx.Err() 161 } 162 } 163 }) 164 165 g2.Go(func() error { 166 for { 167 select { 168 case bundle, ok := <-s.fileBundlesChan: 169 if !ok { 170 return nil 171 } 172 err := s.readAndConvertContentBundle(bundle) 173 if err != nil { 174 return err 175 } 176 case <-ctx.Done(): 177 return ctx.Err() 178 } 179 } 180 }) 181 } 182 183 err := g2.Wait() 184 185 close(s.pagesChan) 186 187 if err != nil { 188 return err 189 } 190 191 if err := g1.Wait(); err != nil { 192 return err 193 } 194 195 s.site.rawAllPages.Sort() 196 197 return nil 198 199 } 200 201 func (s *siteContentProcessor) readAndConvertContentFile(file *fileInfo) error { 202 ctx := &handlerContext{source: file, pages: s.pagesChan} 203 return s.handleContent(ctx).err 204 } 205 206 func (s *siteContentProcessor) readAndConvertContentBundle(bundle *bundleDir) error { 207 ctx := &handlerContext{bundle: bundle, pages: s.pagesChan} 208 return s.handleContent(ctx).err 209 }