github.com/graemephi/kahugo@v0.62.3-0.20211121071557-d78c0423784d/hugolib/hugo_sites_build.go (about) 1 // Copyright 2019 The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package hugolib 15 16 import ( 17 "bytes" 18 "context" 19 "encoding/json" 20 "fmt" 21 "os" 22 "path/filepath" 23 "runtime/trace" 24 "strings" 25 26 "github.com/gohugoio/hugo/publisher" 27 28 "github.com/gohugoio/hugo/hugofs" 29 30 "github.com/gohugoio/hugo/common/para" 31 "github.com/gohugoio/hugo/config" 32 "github.com/gohugoio/hugo/resources/postpub" 33 34 "github.com/spf13/afero" 35 36 "github.com/gohugoio/hugo/output" 37 38 "github.com/pkg/errors" 39 40 "github.com/fsnotify/fsnotify" 41 "github.com/gohugoio/hugo/helpers" 42 ) 43 44 // Build builds all sites. If filesystem events are provided, 45 // this is considered to be a potential partial rebuild. 46 func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { 47 ctx, task := trace.NewTask(context.Background(), "Build") 48 defer task.End() 49 50 if !config.NoBuildLock { 51 unlock, err := h.BaseFs.LockBuild() 52 if err != nil { 53 return errors.Wrap(err, "failed to acquire a build lock") 54 } 55 defer unlock() 56 } 57 58 errCollector := h.StartErrorCollector() 59 errs := make(chan error) 60 61 go func(from, to chan error) { 62 var errors []error 63 i := 0 64 for e := range from { 65 i++ 66 if i > 50 { 67 break 68 } 69 errors = append(errors, e) 70 } 71 to <- h.pickOneAndLogTheRest(errors) 72 73 close(to) 74 }(errCollector, errs) 75 76 if h.Metrics != nil { 77 h.Metrics.Reset() 78 } 79 80 h.testCounters = config.testCounters 81 82 // Need a pointer as this may be modified. 83 conf := &config 84 85 if conf.whatChanged == nil { 86 // Assume everything has changed 87 conf.whatChanged = &whatChanged{source: true} 88 } 89 90 var prepareErr error 91 92 if !config.PartialReRender { 93 prepare := func() error { 94 init := func(conf *BuildCfg) error { 95 for _, s := range h.Sites { 96 s.Deps.BuildStartListeners.Notify() 97 } 98 99 if len(events) > 0 { 100 // Rebuild 101 if err := h.initRebuild(conf); err != nil { 102 return errors.Wrap(err, "initRebuild") 103 } 104 } else { 105 if err := h.initSites(conf); err != nil { 106 return errors.Wrap(err, "initSites") 107 } 108 } 109 110 return nil 111 } 112 113 var err error 114 115 f := func() { 116 err = h.process(conf, init, events...) 117 } 118 trace.WithRegion(ctx, "process", f) 119 if err != nil { 120 return errors.Wrap(err, "process") 121 } 122 123 f = func() { 124 err = h.assemble(conf) 125 } 126 trace.WithRegion(ctx, "assemble", f) 127 if err != nil { 128 return err 129 } 130 131 return nil 132 } 133 134 f := func() { 135 prepareErr = prepare() 136 } 137 trace.WithRegion(ctx, "prepare", f) 138 if prepareErr != nil { 139 h.SendError(prepareErr) 140 } 141 142 } 143 144 if prepareErr == nil { 145 var err error 146 f := func() { 147 err = h.render(conf) 148 } 149 trace.WithRegion(ctx, "render", f) 150 if err != nil { 151 h.SendError(err) 152 } 153 154 if err = h.postProcess(); err != nil { 155 h.SendError(err) 156 } 157 } 158 159 if h.Metrics != nil { 160 var b bytes.Buffer 161 h.Metrics.WriteMetrics(&b) 162 163 h.Log.Printf("\nTemplate Metrics:\n\n") 164 h.Log.Println(b.String()) 165 } 166 167 select { 168 // Make sure the channel always gets something. 169 case errCollector <- nil: 170 default: 171 } 172 close(errCollector) 173 174 err := <-errs 175 if err != nil { 176 return err 177 } 178 179 if err := h.fatalErrorHandler.getErr(); err != nil { 180 return err 181 } 182 183 errorCount := h.Log.LogCounters().ErrorCounter.Count() 184 if errorCount > 0 { 185 return fmt.Errorf("logged %d error(s)", errorCount) 186 } 187 188 return nil 189 } 190 191 // Build lifecycle methods below. 192 // The order listed matches the order of execution. 193 194 func (h *HugoSites) initSites(config *BuildCfg) error { 195 h.reset(config) 196 197 if config.NewConfig != nil { 198 if err := h.createSitesFromConfig(config.NewConfig); err != nil { 199 return err 200 } 201 } 202 203 return nil 204 } 205 206 func (h *HugoSites) initRebuild(config *BuildCfg) error { 207 if config.NewConfig != nil { 208 return errors.New("rebuild does not support 'NewConfig'") 209 } 210 211 if config.ResetState { 212 return errors.New("rebuild does not support 'ResetState'") 213 } 214 215 if !h.running { 216 return errors.New("rebuild called when not in watch mode") 217 } 218 219 for _, s := range h.Sites { 220 s.resetBuildState(config.whatChanged.source) 221 } 222 223 h.reset(config) 224 h.resetLogs() 225 helpers.InitLoggers() 226 227 return nil 228 } 229 230 func (h *HugoSites) process(config *BuildCfg, init func(config *BuildCfg) error, events ...fsnotify.Event) error { 231 // We should probably refactor the Site and pull up most of the logic from there to here, 232 // but that seems like a daunting task. 233 // So for now, if there are more than one site (language), 234 // we pre-process the first one, then configure all the sites based on that. 235 236 firstSite := h.Sites[0] 237 238 if len(events) > 0 { 239 // This is a rebuild 240 return firstSite.processPartial(config, init, events) 241 } 242 243 return firstSite.process(*config) 244 } 245 246 func (h *HugoSites) assemble(bcfg *BuildCfg) error { 247 if len(h.Sites) > 1 { 248 // The first is initialized during process; initialize the rest 249 for _, site := range h.Sites[1:] { 250 if err := site.initializeSiteInfo(); err != nil { 251 return err 252 } 253 } 254 } 255 256 if !bcfg.whatChanged.source { 257 return nil 258 } 259 260 if err := h.getContentMaps().AssemblePages(); err != nil { 261 return err 262 } 263 264 if err := h.createPageCollections(); err != nil { 265 return err 266 } 267 268 return nil 269 } 270 271 func (h *HugoSites) render(config *BuildCfg) error { 272 if _, err := h.init.layouts.Do(); err != nil { 273 return err 274 } 275 276 siteRenderContext := &siteRenderContext{cfg: config, multihost: h.multihost} 277 278 if !config.PartialReRender { 279 h.renderFormats = output.Formats{} 280 h.withSite(func(s *Site) error { 281 s.initRenderFormats() 282 return nil 283 }) 284 285 for _, s := range h.Sites { 286 h.renderFormats = append(h.renderFormats, s.renderFormats...) 287 } 288 } 289 290 i := 0 291 for _, s := range h.Sites { 292 for siteOutIdx, renderFormat := range s.renderFormats { 293 siteRenderContext.outIdx = siteOutIdx 294 siteRenderContext.sitesOutIdx = i 295 i++ 296 297 select { 298 case <-h.Done(): 299 return nil 300 default: 301 for _, s2 := range h.Sites { 302 // We render site by site, but since the content is lazily rendered 303 // and a site can "borrow" content from other sites, every site 304 // needs this set. 305 s2.rc = &siteRenderingContext{Format: renderFormat} 306 307 if err := s2.preparePagesForRender(s == s2, siteRenderContext.sitesOutIdx); err != nil { 308 return err 309 } 310 } 311 312 if !config.SkipRender { 313 if config.PartialReRender { 314 if err := s.renderPages(siteRenderContext); err != nil { 315 return err 316 } 317 } else { 318 if err := s.render(siteRenderContext); err != nil { 319 return err 320 } 321 } 322 } 323 } 324 325 } 326 } 327 328 if !config.SkipRender { 329 if err := h.renderCrossSitesSitemap(); err != nil { 330 return err 331 } 332 if err := h.renderCrossSitesRobotsTXT(); err != nil { 333 return err 334 } 335 } 336 337 return nil 338 } 339 340 func (h *HugoSites) postProcess() error { 341 // Make sure to write any build stats to disk first so it's available 342 // to the post processors. 343 if err := h.writeBuildStats(); err != nil { 344 return err 345 } 346 347 // This will only be set when js.Build have been triggered with 348 // imports that resolves to the project or a module. 349 // Write a jsconfig.json file to the project's /asset directory 350 // to help JS intellisense in VS Code etc. 351 if !h.ResourceSpec.BuildConfig.NoJSConfigInAssets && h.BaseFs.Assets.Dirs != nil { 352 fi, err := h.BaseFs.Assets.Fs.Stat("") 353 if err != nil { 354 h.Log.Warnf("Failed to resolve jsconfig.json dir: %s", err) 355 } else { 356 m := fi.(hugofs.FileMetaInfo).Meta() 357 assetsDir := m.SourceRoot 358 if strings.HasPrefix(assetsDir, h.ResourceSpec.WorkingDir) { 359 if jsConfig := h.ResourceSpec.JSConfigBuilder.Build(assetsDir); jsConfig != nil { 360 361 b, err := json.MarshalIndent(jsConfig, "", " ") 362 if err != nil { 363 h.Log.Warnf("Failed to create jsconfig.json: %s", err) 364 } else { 365 filename := filepath.Join(assetsDir, "jsconfig.json") 366 if h.running { 367 h.skipRebuildForFilenamesMu.Lock() 368 h.skipRebuildForFilenames[filename] = true 369 h.skipRebuildForFilenamesMu.Unlock() 370 } 371 // Make sure it's written to the OS fs as this is used by 372 // editors. 373 if err := afero.WriteFile(hugofs.Os, filename, b, 0666); err != nil { 374 h.Log.Warnf("Failed to write jsconfig.json: %s", err) 375 } 376 } 377 } 378 } 379 380 } 381 } 382 383 var toPostProcess []postpub.PostPublishedResource 384 for _, r := range h.ResourceSpec.PostProcessResources { 385 toPostProcess = append(toPostProcess, r) 386 } 387 388 if len(toPostProcess) == 0 { 389 // Nothing more to do. 390 return nil 391 } 392 393 workers := para.New(config.GetNumWorkerMultiplier()) 394 g, _ := workers.Start(context.Background()) 395 396 handleFile := func(filename string) error { 397 content, err := afero.ReadFile(h.BaseFs.PublishFs, filename) 398 if err != nil { 399 return err 400 } 401 402 k := 0 403 changed := false 404 405 for { 406 l := bytes.Index(content[k:], []byte(postpub.PostProcessPrefix)) 407 if l == -1 { 408 break 409 } 410 m := bytes.Index(content[k+l:], []byte(postpub.PostProcessSuffix)) + len(postpub.PostProcessSuffix) 411 412 low, high := k+l, k+l+m 413 414 field := content[low:high] 415 416 forward := l + m 417 418 for i, r := range toPostProcess { 419 if r == nil { 420 panic(fmt.Sprintf("resource %d to post process is nil", i+1)) 421 } 422 v, ok := r.GetFieldString(string(field)) 423 if ok { 424 content = append(content[:low], append([]byte(v), content[high:]...)...) 425 changed = true 426 forward = len(v) 427 break 428 } 429 } 430 431 k += forward 432 } 433 434 if changed { 435 return afero.WriteFile(h.BaseFs.PublishFs, filename, content, 0666) 436 } 437 438 return nil 439 } 440 441 _ = afero.Walk(h.BaseFs.PublishFs, "", func(path string, info os.FileInfo, err error) error { 442 if info == nil || info.IsDir() { 443 return nil 444 } 445 446 if !strings.HasSuffix(path, "html") { 447 return nil 448 } 449 450 g.Run(func() error { 451 return handleFile(path) 452 }) 453 454 return nil 455 }) 456 457 // Prepare for a new build. 458 for _, s := range h.Sites { 459 s.ResourceSpec.PostProcessResources = make(map[string]postpub.PostPublishedResource) 460 } 461 462 return g.Wait() 463 } 464 465 type publishStats struct { 466 CSSClasses string `json:"cssClasses"` 467 } 468 469 func (h *HugoSites) writeBuildStats() error { 470 if !h.ResourceSpec.BuildConfig.WriteStats { 471 return nil 472 } 473 474 htmlElements := &publisher.HTMLElements{} 475 for _, s := range h.Sites { 476 stats := s.publisher.PublishStats() 477 htmlElements.Merge(stats.HTMLElements) 478 } 479 480 htmlElements.Sort() 481 482 stats := publisher.PublishStats{ 483 HTMLElements: *htmlElements, 484 } 485 486 js, err := json.MarshalIndent(stats, "", " ") 487 if err != nil { 488 return err 489 } 490 491 filename := filepath.Join(h.WorkingDir, "hugo_stats.json") 492 493 // Make sure it's always written to the OS fs. 494 if err := afero.WriteFile(hugofs.Os, filename, js, 0666); err != nil { 495 return err 496 } 497 498 // Write to the destination, too, if a mem fs is in play. 499 if h.Fs.Source != hugofs.Os { 500 if err := afero.WriteFile(h.Fs.Destination, filename, js, 0666); err != nil { 501 return err 502 } 503 } 504 505 return nil 506 }