github.com/linchen2chris/hugo@v0.0.0-20230307053224-cec209389705/hugolib/integrationtest_builder.go (about) 1 package hugolib 2 3 import ( 4 "bytes" 5 "encoding/base64" 6 "fmt" 7 "io" 8 "os" 9 "path/filepath" 10 "regexp" 11 "strings" 12 "sync" 13 "testing" 14 15 jww "github.com/spf13/jwalterweatherman" 16 17 qt "github.com/frankban/quicktest" 18 "github.com/fsnotify/fsnotify" 19 "github.com/gohugoio/hugo/common/herrors" 20 "github.com/gohugoio/hugo/common/hexec" 21 "github.com/gohugoio/hugo/common/loggers" 22 "github.com/gohugoio/hugo/config" 23 "github.com/gohugoio/hugo/config/security" 24 "github.com/gohugoio/hugo/deps" 25 "github.com/gohugoio/hugo/helpers" 26 "github.com/gohugoio/hugo/htesting" 27 "github.com/gohugoio/hugo/hugofs" 28 "github.com/spf13/afero" 29 "golang.org/x/tools/txtar" 30 ) 31 32 func NewIntegrationTestBuilder(conf IntegrationTestConfig) *IntegrationTestBuilder { 33 // Code fences. 34 conf.TxtarString = strings.ReplaceAll(conf.TxtarString, "§§§", "```") 35 // Multiline strings. 36 conf.TxtarString = strings.ReplaceAll(conf.TxtarString, "§§", "`") 37 38 data := txtar.Parse([]byte(conf.TxtarString)) 39 40 c, ok := conf.T.(*qt.C) 41 if !ok { 42 c = qt.New(conf.T) 43 } 44 45 if conf.NeedsOsFS { 46 if !filepath.IsAbs(conf.WorkingDir) { 47 tempDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-integration-test") 48 c.Assert(err, qt.IsNil) 49 conf.WorkingDir = filepath.Join(tempDir, conf.WorkingDir) 50 if !conf.PrintAndKeepTempDir { 51 c.Cleanup(clean) 52 } else { 53 fmt.Println("\nUsing WorkingDir dir:", conf.WorkingDir) 54 } 55 } 56 } else if conf.WorkingDir == "" { 57 conf.WorkingDir = helpers.FilePathSeparator 58 } 59 60 return &IntegrationTestBuilder{ 61 Cfg: conf, 62 C: c, 63 data: data, 64 } 65 } 66 67 // IntegrationTestBuilder is a (partial) rewrite of sitesBuilder. 68 // The main problem with the "old" one was that it was that the test data was often a little hidden, 69 // so it became hard to look at a test and determine what it should do, especially coming back to the 70 // test after a year or so. 71 type IntegrationTestBuilder struct { 72 *qt.C 73 74 data *txtar.Archive 75 76 fs *hugofs.Fs 77 H *HugoSites 78 79 Cfg IntegrationTestConfig 80 81 changedFiles []string 82 createdFiles []string 83 removedFiles []string 84 renamedFiles []string 85 86 buildCount int 87 GCCount int 88 counters *testCounters 89 logBuff lockingBuffer 90 91 builderInit sync.Once 92 } 93 94 type lockingBuffer struct { 95 sync.Mutex 96 bytes.Buffer 97 } 98 99 func (b *lockingBuffer) Write(p []byte) (n int, err error) { 100 b.Lock() 101 n, err = b.Buffer.Write(p) 102 b.Unlock() 103 return 104 } 105 106 func (s *IntegrationTestBuilder) AssertLogContains(text string) { 107 s.Helper() 108 s.Assert(s.logBuff.String(), qt.Contains, text) 109 } 110 111 func (s *IntegrationTestBuilder) AssertLogMatches(expression string) { 112 s.Helper() 113 re := regexp.MustCompile(expression) 114 s.Assert(re.MatchString(s.logBuff.String()), qt.IsTrue, qt.Commentf(s.logBuff.String())) 115 } 116 117 func (s *IntegrationTestBuilder) AssertBuildCountData(count int) { 118 s.Helper() 119 s.Assert(s.H.init.data.InitCount(), qt.Equals, count) 120 } 121 122 func (s *IntegrationTestBuilder) AssertBuildCountGitInfo(count int) { 123 s.Helper() 124 s.Assert(s.H.init.gitInfo.InitCount(), qt.Equals, count) 125 } 126 127 func (s *IntegrationTestBuilder) AssertBuildCountLayouts(count int) { 128 s.Helper() 129 s.Assert(s.H.init.layouts.InitCount(), qt.Equals, count) 130 } 131 132 func (s *IntegrationTestBuilder) AssertBuildCountTranslations(count int) { 133 s.Helper() 134 s.Assert(s.H.init.translations.InitCount(), qt.Equals, count) 135 } 136 137 func (s *IntegrationTestBuilder) AssertFileContent(filename string, matches ...string) { 138 s.Helper() 139 content := strings.TrimSpace(s.FileContent(filename)) 140 for _, m := range matches { 141 lines := strings.Split(m, "\n") 142 for _, match := range lines { 143 match = strings.TrimSpace(match) 144 if match == "" || strings.HasPrefix(match, "#") { 145 continue 146 } 147 s.Assert(content, qt.Contains, match, qt.Commentf(m)) 148 } 149 } 150 } 151 152 func (s *IntegrationTestBuilder) AssertFileContentExact(filename string, matches ...string) { 153 s.Helper() 154 content := s.FileContent(filename) 155 for _, m := range matches { 156 s.Assert(content, qt.Contains, m, qt.Commentf(m)) 157 } 158 } 159 160 func (s *IntegrationTestBuilder) AssertDestinationExists(filename string, b bool) { 161 checker := qt.IsTrue 162 if !b { 163 checker = qt.IsFalse 164 } 165 s.Assert(s.destinationExists(filepath.Clean(filename)), checker) 166 } 167 168 func (s *IntegrationTestBuilder) destinationExists(filename string) bool { 169 b, err := helpers.Exists(filename, s.fs.PublishDir) 170 if err != nil { 171 panic(err) 172 } 173 return b 174 } 175 176 func (s *IntegrationTestBuilder) AssertIsFileError(err error) herrors.FileError { 177 s.Assert(err, qt.ErrorAs, new(herrors.FileError)) 178 return herrors.UnwrapFileError(err) 179 } 180 181 func (s *IntegrationTestBuilder) AssertRenderCountContent(count int) { 182 s.Helper() 183 s.Assert(s.counters.contentRenderCounter, qt.Equals, uint64(count)) 184 } 185 186 func (s *IntegrationTestBuilder) AssertRenderCountPage(count int) { 187 s.Helper() 188 s.Assert(s.counters.pageRenderCounter, qt.Equals, uint64(count)) 189 } 190 191 func (s *IntegrationTestBuilder) Build() *IntegrationTestBuilder { 192 s.Helper() 193 _, err := s.BuildE() 194 if s.Cfg.Verbose || err != nil { 195 fmt.Println(s.logBuff.String()) 196 } 197 if s.Cfg.RunGC { 198 s.GCCount, err = s.H.GC() 199 } 200 s.Assert(err, qt.IsNil) 201 return s 202 } 203 204 func (s *IntegrationTestBuilder) BuildE() (*IntegrationTestBuilder, error) { 205 s.Helper() 206 if err := s.initBuilder(); err != nil { 207 return s, err 208 } 209 210 err := s.build(BuildCfg{}) 211 return s, err 212 } 213 214 type IntegrationTestDebugConfig struct { 215 Out io.Writer 216 217 PrintDestinationFs bool 218 PrintPagemap bool 219 220 PrefixDestinationFs string 221 PrefixPagemap string 222 } 223 224 func (s *IntegrationTestBuilder) EditFileReplace(filename string, replacementFunc func(s string) string) *IntegrationTestBuilder { 225 absFilename := s.absFilename(filename) 226 b, err := afero.ReadFile(s.fs.Source, absFilename) 227 s.Assert(err, qt.IsNil) 228 s.changedFiles = append(s.changedFiles, absFilename) 229 oldContent := string(b) 230 s.writeSource(absFilename, replacementFunc(oldContent)) 231 return s 232 } 233 234 func (s *IntegrationTestBuilder) EditFiles(filenameContent ...string) *IntegrationTestBuilder { 235 for i := 0; i < len(filenameContent); i += 2 { 236 filename, content := filepath.FromSlash(filenameContent[i]), filenameContent[i+1] 237 absFilename := s.absFilename(filename) 238 s.changedFiles = append(s.changedFiles, absFilename) 239 s.writeSource(absFilename, content) 240 } 241 return s 242 } 243 244 func (s *IntegrationTestBuilder) AddFiles(filenameContent ...string) *IntegrationTestBuilder { 245 for i := 0; i < len(filenameContent); i += 2 { 246 filename, content := filepath.FromSlash(filenameContent[i]), filenameContent[i+1] 247 absFilename := s.absFilename(filename) 248 s.createdFiles = append(s.createdFiles, absFilename) 249 s.writeSource(absFilename, content) 250 } 251 return s 252 } 253 254 func (s *IntegrationTestBuilder) RemoveFiles(filenames ...string) *IntegrationTestBuilder { 255 for _, filename := range filenames { 256 absFilename := s.absFilename(filename) 257 s.removedFiles = append(s.removedFiles, absFilename) 258 s.Assert(s.fs.Source.Remove(absFilename), qt.IsNil) 259 260 } 261 262 return s 263 } 264 265 func (s *IntegrationTestBuilder) RenameFile(old, new string) *IntegrationTestBuilder { 266 absOldFilename := s.absFilename(old) 267 absNewFilename := s.absFilename(new) 268 s.renamedFiles = append(s.renamedFiles, absOldFilename) 269 s.createdFiles = append(s.createdFiles, absNewFilename) 270 s.Assert(s.fs.Source.MkdirAll(filepath.Dir(absNewFilename), 0777), qt.IsNil) 271 s.Assert(s.fs.Source.Rename(absOldFilename, absNewFilename), qt.IsNil) 272 return s 273 } 274 275 func (s *IntegrationTestBuilder) FileContent(filename string) string { 276 s.Helper() 277 return s.readWorkingDir(s, s.fs, filepath.FromSlash(filename)) 278 } 279 280 func (s *IntegrationTestBuilder) initBuilder() error { 281 var initErr error 282 s.builderInit.Do(func() { 283 var afs afero.Fs 284 if s.Cfg.NeedsOsFS { 285 afs = afero.NewOsFs() 286 } else { 287 afs = afero.NewMemMapFs() 288 } 289 290 if s.Cfg.LogLevel == 0 { 291 s.Cfg.LogLevel = jww.LevelWarn 292 } 293 294 logger := loggers.NewBasicLoggerForWriter(s.Cfg.LogLevel, &s.logBuff) 295 296 isBinaryRe := regexp.MustCompile(`^(.*)(\.png|\.jpg)$`) 297 298 for _, f := range s.data.Files { 299 filename := filepath.Join(s.Cfg.WorkingDir, f.Name) 300 data := bytes.TrimSuffix(f.Data, []byte("\n")) 301 if isBinaryRe.MatchString(filename) { 302 var err error 303 data, err = base64.StdEncoding.DecodeString(string(data)) 304 s.Assert(err, qt.IsNil) 305 306 } 307 s.Assert(afs.MkdirAll(filepath.Dir(filename), 0777), qt.IsNil) 308 s.Assert(afero.WriteFile(afs, filename, data, 0666), qt.IsNil) 309 } 310 311 configDirFilename := filepath.Join(s.Cfg.WorkingDir, "config") 312 if _, err := afs.Stat(configDirFilename); err != nil { 313 configDirFilename = "" 314 } 315 316 cfg, _, err := LoadConfig( 317 ConfigSourceDescriptor{ 318 WorkingDir: s.Cfg.WorkingDir, 319 AbsConfigDir: configDirFilename, 320 Fs: afs, 321 Logger: logger, 322 Environ: []string{}, 323 }, 324 func(cfg config.Provider) error { 325 return nil 326 }, 327 ) 328 329 s.Assert(err, qt.IsNil) 330 331 cfg.Set("workingDir", s.Cfg.WorkingDir) 332 333 fs := hugofs.NewFrom(afs, cfg) 334 335 s.Assert(err, qt.IsNil) 336 337 depsCfg := deps.DepsCfg{Cfg: cfg, Fs: fs, Running: s.Cfg.Running, Logger: logger} 338 sites, err := NewHugoSites(depsCfg) 339 if err != nil { 340 initErr = err 341 return 342 } 343 344 s.H = sites 345 s.fs = fs 346 347 if s.Cfg.NeedsNpmInstall { 348 wd, _ := os.Getwd() 349 s.Assert(os.Chdir(s.Cfg.WorkingDir), qt.IsNil) 350 s.C.Cleanup(func() { os.Chdir(wd) }) 351 sc := security.DefaultConfig 352 sc.Exec.Allow = security.NewWhitelist("npm") 353 ex := hexec.New(sc) 354 command, err := ex.New("npm", "install") 355 s.Assert(err, qt.IsNil) 356 s.Assert(command.Run(), qt.IsNil) 357 358 } 359 }) 360 361 return initErr 362 } 363 364 func (s *IntegrationTestBuilder) absFilename(filename string) string { 365 filename = filepath.FromSlash(filename) 366 if filepath.IsAbs(filename) { 367 return filename 368 } 369 if s.Cfg.WorkingDir != "" && !strings.HasPrefix(filename, s.Cfg.WorkingDir) { 370 filename = filepath.Join(s.Cfg.WorkingDir, filename) 371 } 372 return filename 373 } 374 375 func (s *IntegrationTestBuilder) build(cfg BuildCfg) error { 376 s.Helper() 377 defer func() { 378 s.changedFiles = nil 379 s.createdFiles = nil 380 s.removedFiles = nil 381 s.renamedFiles = nil 382 }() 383 384 changeEvents := s.changeEvents() 385 s.logBuff.Reset() 386 s.counters = &testCounters{} 387 cfg.testCounters = s.counters 388 389 if s.buildCount > 0 && (len(changeEvents) == 0) { 390 return nil 391 } 392 393 s.buildCount++ 394 395 err := s.H.Build(cfg, changeEvents...) 396 if err != nil { 397 return err 398 } 399 logErrorCount := s.H.NumLogErrors() 400 if logErrorCount > 0 { 401 return fmt.Errorf("logged %d error(s): %s", logErrorCount, s.logBuff.String()) 402 } 403 404 return nil 405 } 406 407 func (s *IntegrationTestBuilder) changeEvents() []fsnotify.Event { 408 var events []fsnotify.Event 409 for _, v := range s.removedFiles { 410 events = append(events, fsnotify.Event{ 411 Name: v, 412 Op: fsnotify.Remove, 413 }) 414 } 415 for _, v := range s.renamedFiles { 416 events = append(events, fsnotify.Event{ 417 Name: v, 418 Op: fsnotify.Rename, 419 }) 420 } 421 for _, v := range s.changedFiles { 422 events = append(events, fsnotify.Event{ 423 Name: v, 424 Op: fsnotify.Write, 425 }) 426 } 427 for _, v := range s.createdFiles { 428 events = append(events, fsnotify.Event{ 429 Name: v, 430 Op: fsnotify.Create, 431 }) 432 } 433 434 return events 435 } 436 437 func (s *IntegrationTestBuilder) readWorkingDir(t testing.TB, fs *hugofs.Fs, filename string) string { 438 t.Helper() 439 return s.readFileFromFs(t, fs.WorkingDirReadOnly, filename) 440 } 441 442 func (s *IntegrationTestBuilder) readFileFromFs(t testing.TB, fs afero.Fs, filename string) string { 443 t.Helper() 444 filename = filepath.Clean(filename) 445 b, err := afero.ReadFile(fs, filename) 446 if err != nil { 447 // Print some debug info 448 hadSlash := strings.HasPrefix(filename, helpers.FilePathSeparator) 449 start := 0 450 if hadSlash { 451 start = 1 452 } 453 end := start + 1 454 455 parts := strings.Split(filename, helpers.FilePathSeparator) 456 if parts[start] == "work" { 457 end++ 458 } 459 460 s.Assert(err, qt.IsNil) 461 462 } 463 return string(b) 464 } 465 466 func (s *IntegrationTestBuilder) writeSource(filename, content string) { 467 s.Helper() 468 s.writeToFs(s.fs.Source, filename, content) 469 } 470 471 func (s *IntegrationTestBuilder) writeToFs(fs afero.Fs, filename, content string) { 472 s.Helper() 473 if err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte(content), 0755); err != nil { 474 s.Fatalf("Failed to write file: %s", err) 475 } 476 } 477 478 type IntegrationTestConfig struct { 479 T testing.TB 480 481 // The files to use on txtar format, see 482 // https://pkg.go.dev/golang.org/x/exp/cmd/txtar 483 TxtarString string 484 485 // Whether to simulate server mode. 486 Running bool 487 488 // Will print the log buffer after the build 489 Verbose bool 490 491 LogLevel jww.Threshold 492 493 // Whether it needs the real file system (e.g. for js.Build tests). 494 NeedsOsFS bool 495 496 // Whether to run GC after each build. 497 RunGC bool 498 499 // Do not remove the temp dir after the test. 500 PrintAndKeepTempDir bool 501 502 // Whether to run npm install before Build. 503 NeedsNpmInstall bool 504 505 WorkingDir string 506 }