github.com/shohhei1126/hugo@v0.42.2-0.20180623210752-3d5928889ad7/hugolib/page_bundler.go (about) 1 // Copyright 2017-present The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package hugolib 15 16 import ( 17 "fmt" 18 "math" 19 "runtime" 20 21 // Use this until errgroup gets ported to context 22 // See https://github.com/golang/go/issues/19781 23 "golang.org/x/net/context" 24 "golang.org/x/sync/errgroup" 25 ) 26 27 type siteContentProcessor struct { 28 site *Site 29 30 handleContent contentHandler 31 32 ctx context.Context 33 34 // The input file bundles. 35 fileBundlesChan chan *bundleDir 36 37 // The input file singles. 38 fileSinglesChan chan *fileInfo 39 40 // These assets should be just copied to destination. 41 fileAssetsChan chan []pathLangFile 42 43 numWorkers int 44 45 // The output Pages 46 pagesChan chan *Page 47 48 // Used for partial rebuilds (aka. live reload) 49 // Will signal replacement of pages in the site collection. 50 partialBuild bool 51 } 52 53 func (s *siteContentProcessor) processBundle(b *bundleDir) { 54 select { 55 case s.fileBundlesChan <- b: 56 case <-s.ctx.Done(): 57 } 58 } 59 60 func (s *siteContentProcessor) processSingle(fi *fileInfo) { 61 select { 62 case s.fileSinglesChan <- fi: 63 case <-s.ctx.Done(): 64 } 65 } 66 67 func (s *siteContentProcessor) processAssets(assets []pathLangFile) { 68 select { 69 case s.fileAssetsChan <- assets: 70 case <-s.ctx.Done(): 71 } 72 } 73 74 func newSiteContentProcessor(ctx context.Context, partialBuild bool, s *Site) *siteContentProcessor { 75 numWorkers := 12 76 if n := runtime.NumCPU() * 3; n > numWorkers { 77 numWorkers = n 78 } 79 80 numWorkers = int(math.Ceil(float64(numWorkers) / float64(len(s.owner.Sites)))) 81 82 return &siteContentProcessor{ 83 ctx: ctx, 84 partialBuild: partialBuild, 85 site: s, 86 handleContent: newHandlerChain(s), 87 fileBundlesChan: make(chan *bundleDir, numWorkers), 88 fileSinglesChan: make(chan *fileInfo, numWorkers), 89 fileAssetsChan: make(chan []pathLangFile, numWorkers), 90 numWorkers: numWorkers, 91 pagesChan: make(chan *Page, numWorkers), 92 } 93 } 94 95 func (s *siteContentProcessor) closeInput() { 96 close(s.fileSinglesChan) 97 close(s.fileBundlesChan) 98 close(s.fileAssetsChan) 99 } 100 101 func (s *siteContentProcessor) process(ctx context.Context) error { 102 g1, ctx := errgroup.WithContext(ctx) 103 g2, ctx := errgroup.WithContext(ctx) 104 105 // There can be only one of these per site. 106 g1.Go(func() error { 107 for p := range s.pagesChan { 108 if p.s != s.site { 109 panic(fmt.Sprintf("invalid page site: %v vs %v", p.s, s)) 110 } 111 112 if s.partialBuild { 113 s.site.replacePage(p) 114 } else { 115 s.site.addPage(p) 116 } 117 } 118 return nil 119 }) 120 121 for i := 0; i < s.numWorkers; i++ { 122 g2.Go(func() error { 123 for { 124 select { 125 case f, ok := <-s.fileSinglesChan: 126 if !ok { 127 return nil 128 } 129 err := s.readAndConvertContentFile(f) 130 if err != nil { 131 return err 132 } 133 case <-ctx.Done(): 134 return ctx.Err() 135 } 136 } 137 }) 138 139 g2.Go(func() error { 140 for { 141 select { 142 case files, ok := <-s.fileAssetsChan: 143 if !ok { 144 return nil 145 } 146 for _, file := range files { 147 f, err := s.site.BaseFs.ContentFs.Open(file.Filename()) 148 if err != nil { 149 return fmt.Errorf("failed to open assets file: %s", err) 150 } 151 err = s.site.publish(&s.site.PathSpec.ProcessingStats.Files, file.Path(), f) 152 f.Close() 153 if err != nil { 154 return err 155 } 156 } 157 158 case <-ctx.Done(): 159 return ctx.Err() 160 } 161 } 162 }) 163 164 g2.Go(func() error { 165 for { 166 select { 167 case bundle, ok := <-s.fileBundlesChan: 168 if !ok { 169 return nil 170 } 171 err := s.readAndConvertContentBundle(bundle) 172 if err != nil { 173 return err 174 } 175 case <-ctx.Done(): 176 return ctx.Err() 177 } 178 } 179 }) 180 } 181 182 err := g2.Wait() 183 184 close(s.pagesChan) 185 186 if err != nil { 187 return err 188 } 189 190 if err := g1.Wait(); err != nil { 191 return err 192 } 193 194 s.site.rawAllPages.Sort() 195 196 return nil 197 198 } 199 200 func (s *siteContentProcessor) readAndConvertContentFile(file *fileInfo) error { 201 ctx := &handlerContext{source: file, pages: s.pagesChan} 202 return s.handleContent(ctx).err 203 } 204 205 func (s *siteContentProcessor) readAndConvertContentBundle(bundle *bundleDir) error { 206 ctx := &handlerContext{bundle: bundle, pages: s.pagesChan} 207 return s.handleContent(ctx).err 208 }