github.com/Azareal/Gosora@v0.0.0-20210729070923-553e66b59003/main.go (about) 1 /* 2 * 3 * Gosora Main File 4 * Copyright Azareal 2016 - 2020 5 * 6 */ 7 // Package main contains the main initialisation logic for Gosora 8 package main // import "github.com/Azareal/Gosora" 9 10 import ( 11 "bytes" 12 "crypto/tls" 13 "flag" 14 "fmt" 15 "io" 16 "log" 17 "mime" 18 "net/http" 19 "os" 20 "os/signal" 21 "runtime" 22 "runtime/debug" 23 "runtime/pprof" 24 "strconv" 25 "strings" 26 "syscall" 27 "time" 28 29 c "github.com/Azareal/Gosora/common" 30 co "github.com/Azareal/Gosora/common/counters" 31 meta "github.com/Azareal/Gosora/common/meta" 32 p "github.com/Azareal/Gosora/common/phrases" 33 _ "github.com/Azareal/Gosora/extend" 34 qgen "github.com/Azareal/Gosora/query_gen" 35 "github.com/Azareal/Gosora/routes" 36 "github.com/Azareal/Gosora/uutils" 37 "github.com/fsnotify/fsnotify" 38 39 //"github.com/lucas-clemente/quic-go/http3" 40 "github.com/pkg/errors" 41 ) 42 43 var router *GenRouter 44 45 // TODO: Wrap the globals in here so we can pass pointers to them to subpackages 46 var globs *Globs 47 48 type Globs struct { 49 stmts *Stmts 50 } 51 52 // Temporary alias for renderTemplate 53 func init() { 54 c.RenderTemplateAlias = routes.RenderTemplate 55 } 56 57 func afterDBInit() (err error) { 58 if err := storeInit(); err != nil { 59 return err 60 } 61 log.Print("Exitted storeInit") 62 63 c.GzipStartEtag = "\"" + strconv.FormatInt(c.StartTime.Unix(), 10) + "-ng\"" 64 c.StartEtag = "\"" + strconv.FormatInt(c.StartTime.Unix(), 10) + "-n\"" 65 66 var uids []int 67 tc := c.Topics.GetCache() 68 if tc != nil { 69 log.Print("Preloading topics") 70 // Preload ten topics to get the wheels going 71 var count = 10 72 if tc.GetCapacity() <= 10 { 73 count = 2 74 if tc.GetCapacity() <= 2 { 75 count = 0 76 } 77 } 78 group, err := c.Groups.Get(c.GuestUser.Group) 79 if err != nil { 80 return err 81 } 82 83 // TODO: Use the same cached data for both the topic list and the topic fetches... 84 tList, _, _, err := c.TopicList.GetListByCanSee(group.CanSee, 1, 0, nil) 85 if err != nil { 86 return err 87 } 88 ctList := make([]*c.TopicsRow, len(tList)) 89 copy(ctList, tList) 90 91 tList, _, _, err = c.TopicList.GetListByCanSee(group.CanSee, 2, 0, nil) 92 if err != nil { 93 return err 94 } 95 for _, tItem := range tList { 96 ctList = append(ctList, tItem) 97 } 98 99 tList, _, _, err = c.TopicList.GetListByCanSee(group.CanSee, 3, 0, nil) 100 if err != nil { 101 return err 102 } 103 for _, tItem := range tList { 104 ctList = append(ctList, tItem) 105 } 106 107 if count > len(ctList) { 108 count = len(ctList) 109 } 110 for i := 0; i < count; i++ { 111 _, _ = c.Topics.Get(ctList[i].ID) 112 } 113 } 114 115 uc := c.Users.GetCache() 116 if uc != nil { 117 // Preload associated users too... 118 for _, uid := range uids { 119 _, _ = c.Users.Get(uid) 120 } 121 } 122 123 log.Print("Exitted afterDBInit") 124 return nil 125 } 126 127 // Experimenting with a new error package here to try to reduce the amount of debugging we have to do 128 // TODO: Dynamically register these items to avoid maintaining as much code here? 129 func storeInit() (e error) { 130 acc := qgen.NewAcc() 131 ws := errors.WithStack 132 var rcache c.ReplyCache 133 if c.Config.ReplyCache == "static" { 134 rcache = c.NewMemoryReplyCache(c.Config.ReplyCacheCapacity) 135 } 136 c.Rstore, e = c.NewSQLReplyStore(acc, rcache) 137 if e != nil { 138 return ws(e) 139 } 140 c.Prstore, e = c.NewSQLProfileReplyStore(acc) 141 if e != nil { 142 return ws(e) 143 } 144 c.Likes, e = c.NewDefaultLikeStore(acc) 145 if e != nil { 146 return ws(e) 147 } 148 c.ForumActionStore, e = c.NewDefaultForumActionStore(acc) 149 if e != nil { 150 return ws(e) 151 } 152 c.Convos, e = c.NewDefaultConversationStore(acc) 153 if e != nil { 154 return ws(e) 155 } 156 c.UserBlocks, e = c.NewDefaultBlockStore(acc) 157 if e != nil { 158 return ws(e) 159 } 160 c.GroupPromotions, e = c.NewDefaultGroupPromotionStore(acc) 161 if e != nil { 162 return ws(e) 163 } 164 165 if e = p.InitPhrases(c.Site.Language); e != nil { 166 return ws(e) 167 } 168 if e = c.InitEmoji(); e != nil { 169 return ws(e) 170 } 171 if e = c.InitWeakPasswords(); e != nil { 172 return ws(e) 173 } 174 175 log.Print("Loading the static files.") 176 if e = c.Themes.LoadStaticFiles(); e != nil { 177 return ws(e) 178 } 179 if e = c.StaticFiles.Init(); e != nil { 180 return ws(e) 181 } 182 if e = c.StaticFiles.JSTmplInit(); e != nil { 183 return ws(e) 184 } 185 186 log.Print("Initialising the widgets") 187 c.Widgets = c.NewDefaultWidgetStore() 188 if e = c.InitWidgets(); e != nil { 189 return ws(e) 190 } 191 192 log.Print("Initialising the menu item list") 193 c.Menus = c.NewDefaultMenuStore() 194 if e = c.Menus.Load(1); e != nil { // 1 = the default menu 195 return ws(e) 196 } 197 menuHold, e := c.Menus.Get(1) 198 if e != nil { 199 return ws(e) 200 } 201 fmt.Printf("menuHold: %+v\n", menuHold) 202 var b bytes.Buffer 203 menuHold.Build(&b, &c.GuestUser, "/") 204 fmt.Println("menuHold output: ", string(b.Bytes())) 205 206 log.Print("Initialising the authentication system") 207 c.Auth, e = c.NewDefaultAuth() 208 if e != nil { 209 return ws(e) 210 } 211 212 log.Print("Initialising the stores") 213 c.WordFilters, e = c.NewDefaultWordFilterStore(acc) 214 if e != nil { 215 return ws(e) 216 } 217 c.MFAstore, e = c.NewSQLMFAStore(acc) 218 if e != nil { 219 return ws(e) 220 } 221 c.Pages, e = c.NewDefaultPageStore(acc) 222 if e != nil { 223 return ws(e) 224 } 225 c.Reports, e = c.NewDefaultReportStore(acc) 226 if e != nil { 227 return ws(e) 228 } 229 c.Emails, e = c.NewDefaultEmailStore(acc) 230 if e != nil { 231 return ws(e) 232 } 233 c.LoginLogs, e = c.NewLoginLogStore(acc) 234 if e != nil { 235 return ws(e) 236 } 237 c.RegLogs, e = c.NewRegLogStore(acc) 238 if e != nil { 239 return ws(e) 240 } 241 c.ModLogs, e = c.NewModLogStore(acc) 242 if e != nil { 243 return ws(e) 244 } 245 c.AdminLogs, e = c.NewAdminLogStore(acc) 246 if e != nil { 247 return ws(e) 248 } 249 c.IPSearch, e = c.NewDefaultIPSearcher() 250 if e != nil { 251 return ws(e) 252 } 253 if c.Config.Search == "" || c.Config.Search == "sql" { 254 c.RepliesSearch, e = c.NewSQLSearcher(acc) 255 if e != nil { 256 return ws(e) 257 } 258 } 259 c.Subscriptions, e = c.NewDefaultSubscriptionStore() 260 if e != nil { 261 return ws(e) 262 } 263 c.Attachments, e = c.NewDefaultAttachmentStore(acc) 264 if e != nil { 265 return ws(e) 266 } 267 c.Polls, e = c.NewDefaultPollStore(c.NewMemoryPollCache(100)) // TODO: Max number of polls held in cache, make this a config item 268 if e != nil { 269 return ws(e) 270 } 271 c.TopicList, e = c.NewDefaultTopicList(acc) 272 if e != nil { 273 return ws(e) 274 } 275 c.PasswordResetter, e = c.NewDefaultPasswordResetter(acc) 276 if e != nil { 277 return ws(e) 278 } 279 c.Analytics = c.NewDefaultAnalytics() 280 c.Activity, e = c.NewDefaultActivityStream(acc) 281 if e != nil { 282 return ws(e) 283 } 284 c.ActivityMatches, e = c.NewDefaultActivityStreamMatches(acc) 285 if e != nil { 286 return ws(e) 287 } 288 // TODO: Let the admin choose other thumbnailers, maybe ones defined in plugins 289 c.Thumbnailer = c.NewCaireThumbnailer() 290 c.Recalc, e = c.NewDefaultRecalc(acc) 291 if e != nil { 292 return ws(e) 293 } 294 295 log.Print("Initialising the meta store") 296 c.Meta, e = meta.NewDefaultMetaStore(acc) 297 if e != nil { 298 return ws(e) 299 } 300 301 log.Print("Initialising the view counters") 302 if !c.Config.DisableAnalytics { 303 co.GlobalViewCounter, e = co.NewGlobalViewCounter(acc) 304 if e != nil { 305 return ws(e) 306 } 307 co.AgentViewCounter, e = co.NewDefaultAgentViewCounter(acc) 308 if e != nil { 309 return ws(e) 310 } 311 co.OSViewCounter, e = co.NewDefaultOSViewCounter(acc) 312 if e != nil { 313 return ws(e) 314 } 315 co.LangViewCounter, e = co.NewDefaultLangViewCounter(acc) 316 if e != nil { 317 return ws(e) 318 } 319 if !c.Config.RefNoTrack { 320 co.ReferrerTracker, e = co.NewDefaultReferrerTracker() 321 if e != nil { 322 return ws(e) 323 } 324 } 325 co.MemoryCounter, e = co.NewMemoryCounter(acc) 326 if e != nil { 327 return ws(e) 328 } 329 co.PerfCounter, e = co.NewDefaultPerfCounter(acc) 330 if e != nil { 331 return ws(e) 332 } 333 } 334 co.RouteViewCounter, e = co.NewDefaultRouteViewCounter(acc) 335 if e != nil { 336 return ws(e) 337 } 338 co.PostCounter, e = co.NewPostCounter() 339 if e != nil { 340 return ws(e) 341 } 342 co.TopicCounter, e = co.NewTopicCounter() 343 if e != nil { 344 return ws(e) 345 } 346 co.TopicViewCounter, e = co.NewDefaultTopicViewCounter() 347 if e != nil { 348 return ws(e) 349 } 350 co.ForumViewCounter, e = co.NewDefaultForumViewCounter() 351 if e != nil { 352 return ws(e) 353 } 354 355 return nil 356 } 357 358 // TODO: Split this function up 359 func main() { 360 // TODO: Recover from panics 361 defer func() { 362 if r := recover(); r != nil { 363 log.Print(r) 364 debug.PrintStack() 365 log.Fatal("Fatal error.") 366 } 367 }() 368 c.StartTime = time.Now() 369 370 // TODO: Have a file for each run with the time/date the server started as the file name? 371 // TODO: Log panics with recover() 372 f, err := os.OpenFile("./logs/ops-"+strconv.FormatInt(c.StartTime.Unix(), 10)+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755) 373 if err != nil { 374 log.Fatal(err) 375 } 376 //c.LogWriter = io.MultiWriter(os.Stderr, f) 377 c.LogWriter = io.MultiWriter(os.Stdout, f) 378 c.ErrLogWriter = io.MultiWriter(os.Stderr, f) 379 log.SetOutput(c.LogWriter) 380 c.ErrLogger = log.New(c.ErrLogWriter, "", log.LstdFlags) 381 log.Print("Running Gosora v" + c.SoftwareVersion.String()) 382 fmt.Println("") 383 384 // TODO: Add a flag for enabling the profiler 385 if false { 386 f, err := os.Create(c.Config.LogDir + "cpu.prof") 387 if err != nil { 388 log.Fatal(err) 389 } 390 pprof.StartCPUProfile(f) 391 } 392 393 err = mime.AddExtensionType(".avif", "image/avif") 394 if err != nil { 395 log.Fatal(err) 396 } 397 398 jsToken, err := c.GenerateSafeString(80) 399 if err != nil { 400 log.Fatal(err) 401 } 402 c.JSTokenBox.Store(jsToken) 403 404 log.Print("Loading the configuration data") 405 err = c.LoadConfig() 406 if err != nil { 407 log.Fatal(err) 408 } 409 log.Print("Processing configuration data") 410 err = c.ProcessConfig() 411 if err != nil { 412 log.Fatal(err) 413 } 414 if c.Config.DisableStdout { 415 c.LogWriter = f 416 log.SetOutput(c.LogWriter) 417 } 418 if c.Config.DisableStderr { 419 c.ErrLogWriter = f 420 c.ErrLogger = log.New(c.ErrLogWriter, "", log.LstdFlags) 421 } 422 c.Tasks = c.NewScheduledTasks() 423 424 err = c.InitTemplates() 425 if err != nil { 426 log.Fatal(err) 427 } 428 c.Themes, err = c.NewThemeList() 429 if err != nil { 430 log.Fatal(err) 431 } 432 c.TopicListThaw = c.NewSingleServerThaw() 433 434 err = InitDatabase() 435 if err != nil { 436 log.Fatal(err) 437 } 438 defer db.Close() 439 440 buildTemplates := flag.Bool("build-templates", false, "build the templates") 441 flag.Parse() 442 if *buildTemplates { 443 if err = c.CompileTemplates(); err != nil { 444 log.Fatal(err) 445 } 446 if err = c.CompileJSTemplates(); err != nil { 447 log.Fatal(err) 448 } 449 return 450 } 451 452 err = afterDBInit() 453 if err != nil { 454 log.Fatalf("%+v", err) 455 } 456 err = c.VerifyConfig() 457 if err != nil { 458 log.Fatal(err) 459 } 460 461 if !c.Dev.NoFsnotify { 462 log.Print("Initialising the file watcher") 463 watcher, err := fsnotify.NewWatcher() 464 if err != nil { 465 log.Fatal(err) 466 } 467 defer watcher.Close() 468 469 go func() { 470 defer c.EatPanics() 471 var ErrFileSkip = errors.New("skip mod file") 472 modifiedFileEvent := func(path string) error { 473 pathBits := strings.Split(path, "\\") 474 if len(pathBits) == 0 { 475 return nil 476 } 477 if pathBits[0] == "themes" { 478 var themeName string 479 if len(pathBits) >= 2 { 480 themeName = pathBits[1] 481 } 482 if len(pathBits) >= 3 && pathBits[2] == "public" { 483 // TODO: Handle new themes freshly plopped into the folder? 484 theme, ok := c.Themes[themeName] 485 if ok { 486 return theme.LoadStaticFiles() 487 } 488 } 489 } 490 return ErrFileSkip 491 } 492 493 // TODO: Expand this to more types of files 494 var err error 495 for { 496 select { 497 case ev := <-watcher.Events: 498 // TODO: Handle file deletes (and renames more graciously by removing the old version of it) 499 if ev.Op&fsnotify.Write == fsnotify.Write { 500 err = modifiedFileEvent(ev.Name) 501 if err != ErrFileSkip { 502 log.Println("modified file:", ev.Name) 503 } else { 504 err = nil 505 } 506 } else if ev.Op&fsnotify.Create == fsnotify.Create { 507 log.Println("new file:", ev.Name) 508 err = modifiedFileEvent(ev.Name) 509 } else { 510 log.Println("unknown event:", ev) 511 err = nil 512 } 513 if err != nil { 514 c.LogError(err) 515 } 516 case err = <-watcher.Errors: 517 c.LogWarning(err) 518 } 519 } 520 }() 521 522 // TODO: Keep tabs on the (non-resource) theme stuff, and the langpacks 523 err = watcher.Add("./public") 524 if err != nil { 525 log.Fatal(err) 526 } 527 err = watcher.Add("./templates") 528 if err != nil { 529 log.Fatal(err) 530 } 531 for _, theme := range c.Themes { 532 err = watcher.Add("./themes/" + theme.Name + "/public") 533 if err != nil { 534 log.Fatal(err) 535 } 536 } 537 } 538 539 /*if err = c.StaticFiles.GenJS(); err != nil { 540 c.LogError(err) 541 }*/ 542 543 log.Print("Checking for init tasks") 544 if err = sched(); err != nil { 545 c.LogError(err) 546 } 547 548 log.Print("Initialising the task system") 549 550 // Thumbnailer goroutine, we only want one image being thumbnailed at a time, otherwise they might wind up consuming all the CPU time and leave no resources left to service the actual requests 551 // TODO: Could we expand this to attachments and other things too? 552 thumbChan := make(chan bool) 553 go c.ThumbTask(thumbChan) 554 if err = tickLoop(thumbChan); err != nil { 555 c.LogError(err) 556 } 557 go TickLoop.Loop() 558 559 // Resource Management Goroutine 560 go func() { 561 defer c.EatPanics() 562 uc, tc := c.Users.GetCache(), c.Topics.GetCache() 563 if uc == nil && tc == nil { 564 return 565 } 566 567 var lastEvictedCount int 568 var couldNotDealloc bool 569 secondTicker := time.NewTicker(time.Second) 570 for { 571 select { 572 case <-secondTicker.C: 573 // TODO: Add a LastRequested field to cached User structs to avoid evicting the same things which wind up getting loaded again anyway? 574 if uc != nil { 575 ucap := uc.GetCapacity() 576 if uc.Length() <= ucap || c.Users.Count() <= ucap { 577 couldNotDealloc = false 578 continue 579 } 580 lastEvictedCount = uc.DeallocOverflow(couldNotDealloc) 581 couldNotDealloc = (lastEvictedCount == 0) 582 } 583 } 584 } 585 }() 586 587 log.Print("Initialising the router") 588 router, err = NewGenRouter(&RouterConfig{ 589 Uploads: http.FileServer(http.Dir("./uploads")), 590 }) 591 if err != nil { 592 log.Fatal(err) 593 } 594 595 log.Print("Initialising the plugins") 596 c.InitPlugins() 597 598 log.Print("Setting up the signal handler") 599 sigs := make(chan os.Signal, 1) 600 signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) 601 go func() { 602 defer c.EatPanics() 603 sig := <-sigs 604 log.Print("Received a signal to shutdown: ", sig) 605 // TODO: Gracefully shutdown the HTTP server 606 tw, cn := c.NewTickWatch(), uutils.Nanotime() 607 tw.Name = "shutdown" 608 tw.Set(&tw.Start, cn) 609 tw.Set(&tw.DBCheck, cn) 610 tw.Run() 611 n, e := func() (string, error) { 612 if e := runHook("before_shutdown_tick"); e != nil { 613 return "before_shutdown_tick ", e 614 } 615 tw.Set(&tw.StartHook, uutils.Nanotime()) 616 log.Print("Running shutdown tasks") 617 if e := c.Tasks.Shutdown.Run(); e != nil { 618 return "shutdown tasks ", e 619 } 620 tw.Set(&tw.Tasks, uutils.Nanotime()) 621 log.Print("Ran shutdown tasks") 622 if e := runHook("after_shutdown_tick"); e != nil { 623 return "after_shutdown_tick ", e 624 } 625 tw.Set(&tw.EndHook, uutils.Nanotime()) 626 return "", nil 627 }() 628 if e != nil { 629 log.Print(n+" err:", e) 630 } 631 tw.Stop() 632 log.Print("Stopping server") 633 c.StoppedServer("Stopped server") 634 }() 635 636 // Start up the WebSocket ticks 637 c.WsHub.Start() 638 639 if false { 640 f, e := os.Create(c.Config.LogDir + "cpu.prof") 641 if e != nil { 642 log.Fatal(e) 643 } 644 pprof.StartCPUProfile(f) 645 } 646 647 //if profiling { 648 // pprof.StopCPUProfile() 649 //} 650 startServer() 651 args := <-c.StopServerChan 652 if false { 653 pprof.StopCPUProfile() 654 f, err := os.Create(c.Config.LogDir + "mem.prof") 655 if err != nil { 656 log.Fatal(err) 657 } 658 defer f.Close() 659 660 runtime.GC() 661 err = pprof.WriteHeapProfile(f) 662 if err != nil { 663 log.Fatal(err) 664 } 665 } 666 // Why did the server stop? 667 log.Fatal(args...) 668 } 669 670 func startServer() { 671 // We might not need timeouts, if we're behind a reverse-proxy like Nginx 672 newServer := func(addr string, h http.Handler) *http.Server { 673 f := func(timeout, dval int) int { 674 if timeout == 0 { 675 timeout = dval 676 } else if timeout == -1 { 677 timeout = 0 678 } 679 return timeout 680 } 681 rtime := f(c.Config.ReadTimeout, 8) 682 wtime := f(c.Config.WriteTimeout, 10) 683 itime := f(c.Config.IdleTimeout, 120) 684 return &http.Server{ 685 Addr: addr, 686 Handler: h, 687 ConnState: c.ConnWatch.StateChange, 688 689 ReadTimeout: time.Duration(rtime) * time.Second, 690 WriteTimeout: time.Duration(wtime) * time.Second, 691 IdleTimeout: time.Duration(itime) * time.Second, 692 693 TLSConfig: &tls.Config{ 694 PreferServerCipherSuites: true, 695 CurvePreferences: []tls.CurveID{ 696 tls.CurveP256, 697 tls.X25519, 698 }, 699 }, 700 } 701 } 702 703 // TODO: Let users run *both* HTTP and HTTPS 704 log.Print("Initialising the HTTP server") 705 /*if c.Dev.QuicPort != 0 { 706 sQuicPort := strconv.Itoa(c.Dev.QuicPort) 707 log.Print("Listening on quic port " + sQuicPort) 708 go func() { 709 defer c.EatPanics() 710 c.StoppedServer(http3.ListenAndServeQUIC(":"+sQuicPort, c.Config.SslFullchain, c.Config.SslPrivkey, router)) 711 }() 712 }*/ 713 714 if !c.Site.EnableSsl { 715 if c.Site.Port == "" { 716 c.Site.Port = "80" 717 } 718 log.Print("Listening on port " + c.Site.Port) 719 go func() { 720 defer c.EatPanics() 721 c.StoppedServer(newServer(":"+c.Site.Port, router).ListenAndServe()) 722 }() 723 return 724 } 725 726 if c.Site.Port == "" { 727 c.Site.Port = "443" 728 } 729 if c.Site.Port == "80" || c.Site.Port == "443" { 730 // We should also run the server on port 80 731 // TODO: Redirect to port 443 732 go func() { 733 defer c.EatPanics() 734 log.Print("Listening on port 80") 735 c.StoppedServer(newServer(":80", &HTTPSRedirect{}).ListenAndServe()) 736 }() 737 } 738 log.Printf("Listening on port %s", c.Site.Port) 739 go func() { 740 defer c.EatPanics() 741 c.StoppedServer(newServer(":"+c.Site.Port, router).ListenAndServeTLS(c.Config.SslFullchain, c.Config.SslPrivkey)) 742 }() 743 }