github.com/Azareal/Gosora@v0.0.0-20210729070923-553e66b59003/router_gen/main.go (about) 1 /* WIP Under Construction */ 2 package main 3 4 import ( 5 "bytes" 6 "log" 7 "os" 8 "strconv" 9 "strings" 10 "text/template" 11 ) 12 13 type TmplVars struct { 14 RouteList []*RouteImpl 15 RouteGroups []*RouteGroup 16 AllRouteNames []RouteName 17 AllRouteMap map[string]int 18 AllAgentNames []string 19 AllAgentMap map[string]int 20 AllAgentMarkNames []string 21 AllAgentMarks map[string]string 22 AllAgentMarkIDs map[string]int 23 AllOSNames []string 24 AllOSMap map[string]int 25 } 26 27 type RouteName struct { 28 Plain string 29 Short string 30 } 31 32 func main() { 33 log.Println("Generating the router...") 34 35 // Load all the routes... 36 r := &Router{} 37 routes(r) 38 39 tmplVars := TmplVars{ 40 RouteList: r.routeList, 41 RouteGroups: r.routeGroups, 42 } 43 var allRouteNames []RouteName 44 allRouteMap := make(map[string]int) 45 46 var out string 47 mapIt := func(name string) { 48 allRouteNames = append(allRouteNames, RouteName{name, strings.Replace(name, "common.", "c.", -1)}) 49 allRouteMap[name] = len(allRouteNames) - 1 50 } 51 mapIt("routes.Error") 52 53 var indentCache [20]string 54 countToIndents := func(ind int) string { 55 out := indentCache[ind] 56 if out != "" { 57 return out 58 } 59 for i := 0; i < ind; i++ { 60 out += "\t" 61 } 62 if ind < 20 { 63 indentCache[ind] = out 64 } 65 return out 66 } 67 o := func(indent int, str string) { 68 out += countToIndents(indent) + str 69 } 70 on := func(indent int, str string) { 71 out += "\n" + countToIndents(indent) + str 72 } 73 iferrn := func(indent int) { 74 ind := countToIndents(indent) 75 ind2 := countToIndents(indent + 1) 76 out += "\n" + ind + "if err != nil {" 77 out += "\n" + ind2 + "return err\n" + ind + "}" 78 } 79 80 runBefore := func(runnables []Runnable, ind int) { 81 if len(runnables) > 0 { 82 for _, runnable := range runnables { 83 if runnable.Literal { 84 on(ind, runnable.Contents) 85 } else { 86 on(ind, "err = c."+runnable.Contents+"(w,req,user)") 87 iferrn(ind) 88 o(ind, "\n") 89 } 90 } 91 } 92 } 93 userCheckNano := func(indent int, route *RouteImpl) { 94 on(indent, "h, err := c.UserCheckNano(w,req,user,cn)") 95 iferrn(indent) 96 vcpy := route.Vars 97 route.Vars = []string{"h"} 98 route.Vars = append(route.Vars, vcpy...) 99 } 100 writeRoute := func(indent int, r *RouteImpl) { 101 on(indent, "err = "+strings.Replace(r.Name, "common.", "c.", -1)+"(w,req,user") 102 for _, item := range r.Vars { 103 out += "," + item 104 } 105 out += `)` 106 } 107 108 for _, route := range r.routeList { 109 mapIt(route.Name) 110 end := len(route.Path) - 1 111 on(2, "case \""+route.Path[0:end]+"\":") 112 //on(3,"id = " + strconv.Itoa(allRouteMap[route.Name])) 113 runBefore(route.RunBefore, 3) 114 if !route.Action && !route.NoHead { 115 userCheckNano(3, route) 116 } 117 writeRoute(3, route) 118 if route.Name != "common.RouteWebsockets" { 119 on(3, "co.RouteViewCounter.Bump3("+strconv.Itoa(allRouteMap[route.Name])+", cn)") 120 } 121 } 122 123 prec := NewPrec() 124 prec.AddSet("MemberOnly", "SuperModOnly", "AdminOnly", "SuperAdminOnly") 125 126 // Hoist runnables which appear on every route to the route group to avoid code duplication 127 dupeMap := make(map[string]int) 128 //skipRunnableAntiDupe: 129 for _, g := range r.routeGroups { 130 for _, route := range g.RouteList { 131 if len(route.RunBefore) == 0 { 132 continue //skipRunnableAntiDupe 133 } 134 // TODO: What if there are duplicates of the same runnable on this route? 135 for _, runnable := range route.RunBefore { 136 dupeMap[runnable.Contents] += 1 137 } 138 } 139 140 // Unset entries which are already set on the route group 141 for _, gRunnable := range g.RunBefore { 142 delete(dupeMap, gRunnable.Contents) 143 for _, item := range prec.LessThanItem(gRunnable.Contents) { 144 delete(dupeMap, item) 145 } 146 } 147 148 for runnable, count := range dupeMap { 149 if count == len(g.RouteList) { 150 g.Before(runnable) 151 } 152 } 153 // This method is optimised in the compiler to do a bulk delete 154 for name, _ := range dupeMap { 155 delete(dupeMap, name) 156 } 157 } 158 159 for _, group := range r.routeGroups { 160 end := len(group.Path) - 1 161 on(2, "case \""+group.Path[0:end]+"\":") 162 runBefore(group.RunBefore, 3) 163 on(3, "switch(req.URL.Path) {") 164 165 defaultRoute := blankRoute() 166 for _, route := range group.RouteList { 167 if group.Path == route.Path { 168 defaultRoute = route 169 continue 170 } 171 mapIt(route.Name) 172 173 on(4, "case \""+route.Path+"\":") 174 //on(5,"id = " + strconv.Itoa(allRouteMap[route.Name])) 175 if len(route.RunBefore) > 0 { 176 skipRunnable: 177 for _, runnable := range route.RunBefore { 178 for _, gRunnable := range group.RunBefore { 179 if gRunnable.Contents == runnable.Contents { 180 continue skipRunnable 181 } 182 if prec.GreaterThan(gRunnable.Contents, runnable.Contents) { 183 continue skipRunnable 184 } 185 } 186 187 if runnable.Literal { 188 on(5, runnable.Contents) 189 } else { 190 on(5, "err = c."+runnable.Contents+"(w,req,user)") 191 iferrn(5) 192 on(5, "") 193 } 194 } 195 } 196 if !route.Action && !route.NoHead && !group.NoHead { 197 userCheckNano(5, route) 198 } 199 writeRoute(5, route) 200 on(5, "co.RouteViewCounter.Bump3("+strconv.Itoa(allRouteMap[route.Name])+", cn)") 201 } 202 203 if defaultRoute.Name != "" { 204 mapIt(defaultRoute.Name) 205 on(4, "default:") 206 //on(5,"id = " + strconv.Itoa(allRouteMap[defaultRoute.Name])) 207 runBefore(defaultRoute.RunBefore, 4) 208 if !defaultRoute.Action && !defaultRoute.NoHead && !group.NoHead { 209 userCheckNano(5, defaultRoute) 210 } 211 writeRoute(5, defaultRoute) 212 on(5, "co.RouteViewCounter.Bump3("+strconv.Itoa(allRouteMap[defaultRoute.Name])+", cn)") 213 } 214 on(3, "}") 215 } 216 217 // Stubs for us to refer to these routes through 218 mapIt("routes.DynamicRoute") 219 mapIt("routes.UploadedFile") 220 mapIt("routes.StaticFile") 221 mapIt("routes.RobotsTxt") 222 mapIt("routes.SitemapXml") 223 mapIt("routes.OpenSearchXml") 224 mapIt("routes.Favicon") 225 mapIt("routes.BadRoute") 226 mapIt("routes.HTTPSRedirect") 227 tmplVars.AllRouteNames = allRouteNames 228 tmplVars.AllRouteMap = allRouteMap 229 230 tmplVars.AllOSNames = []string{ 231 "unknown", 232 "windows", 233 "linux", 234 "mac", 235 "android", 236 "iphone", 237 } 238 tmplVars.AllOSMap = make(map[string]int) 239 for id, os := range tmplVars.AllOSNames { 240 tmplVars.AllOSMap[os] = id 241 } 242 243 tmplVars.AllAgentNames = []string{ 244 "unknown", 245 "opera", 246 "chrome", 247 "firefox", 248 "safari", 249 "edge", 250 "internetexplorer", 251 "trident", // Hack to support IE11 252 253 "androidchrome", 254 "mobilesafari", 255 "samsung", 256 "ucbrowser", 257 258 "googlebot", 259 "yandex", 260 "bing", 261 "slurp", 262 "exabot", 263 "mojeek", 264 "cliqz", 265 "qwant", 266 "datenbank", 267 "baidu", 268 "sogou", 269 "toutiao", 270 "haosou", 271 "duckduckgo", 272 "seznambot", 273 "discord", 274 "telegram", 275 "twitter", 276 "facebook", 277 "cloudflare", 278 "archive_org", 279 "uptimebot", 280 "slackbot", 281 "apple", 282 "discourse", 283 "xenforo", 284 "mattermost", 285 "alexa", 286 "lynx", 287 "blank", 288 "malformed", 289 "suspicious", 290 "semrush", 291 "dotbot", 292 "ahrefs", 293 "proximic", 294 "megaindex", 295 "majestic", 296 "cocolyze", 297 "babbar", 298 "surdotly", 299 "domcop", 300 "netcraft", 301 "seostar", 302 "pandalytics", 303 "blexbot", 304 "wappalyzer", 305 "twingly", 306 "linkfluence", 307 "pagething", 308 "burf", 309 "aspiegel", 310 "mail_ru", 311 "ccbot", 312 "yacy", 313 "zgrab", 314 "cloudsystemnetworks", 315 "maui", 316 "curl", 317 "python", 318 //"go", 319 "headlesschrome", 320 "awesome_bot", 321 } 322 323 tmplVars.AllAgentMap = make(map[string]int) 324 for id, agent := range tmplVars.AllAgentNames { 325 tmplVars.AllAgentMap[agent] = id 326 } 327 328 tmplVars.AllAgentMarkNames = []string{} 329 tmplVars.AllAgentMarks = map[string]string{} 330 331 // Add agent marks 332 a := func(mark, agent string) { 333 tmplVars.AllAgentMarkNames = append(tmplVars.AllAgentMarkNames, mark) 334 tmplVars.AllAgentMarks[mark] = agent 335 } 336 a("OPR", "opera") 337 a("Chrome", "chrome") 338 a("Firefox", "firefox") 339 a("Safari", "safari") 340 a("MSIE", "internetexplorer") 341 a("Trident", "trident") // Hack to support IE11 342 a("Edge", "edge") 343 a("Lynx", "lynx") // There's a rare android variant of lynx which isn't covered by this 344 a("SamsungBrowser", "samsung") 345 a("UCBrowser", "ucbrowser") 346 347 a("Google", "googlebot") 348 a("Googlebot", "googlebot") 349 a("yandex", "yandex") // from the URL 350 a("DuckDuckBot", "duckduckgo") 351 a("DuckDuckGo", "duckduckgo") 352 a("Baiduspider", "baidu") 353 a("Sogou", "sogou") 354 a("ToutiaoSpider", "toutiao") 355 a("Bytespider", "toutiao") 356 a("360Spider", "haosou") 357 a("bingbot", "bing") 358 a("BingPreview", "bing") 359 a("msnbot", "bing") 360 a("Slurp", "slurp") 361 a("Exabot", "exabot") 362 a("MojeekBot", "mojeek") 363 a("Cliqzbot", "cliqz") 364 a("Qwantify", "qwant") 365 a("netEstate", "datenbank") 366 a("SeznamBot", "seznambot") 367 a("CloudFlare", "cloudflare") // Track alwayson specifically in case there are other bots? 368 a("archive", "archive_org") //archive.org_bot 369 a("Uptimebot", "uptimebot") 370 a("Slackbot", "slackbot") 371 a("Slack", "slackbot") 372 a("Discordbot", "discord") 373 a("TelegramBot", "telegram") 374 a("Twitterbot", "twitter") 375 a("facebookexternalhit", "facebook") 376 a("Facebot", "facebook") 377 a("Applebot", "apple") 378 a("Discourse", "discourse") 379 a("XenForo", "xenforo") 380 a("mattermost", "mattermost") 381 a("ia_archiver", "alexa") 382 383 a("SemrushBot", "semrush") 384 a("DotBot", "dotbot") 385 a("AhrefsBot", "ahrefs") 386 a("proximic", "proximic") 387 a("MegaIndex", "megaindex") 388 a("MJ12bot", "majestic") // TODO: This isn't matching bots out in the wild 389 a("mj12bot", "majestic") 390 a("Cocolyzebot", "cocolyze") 391 a("Barkrowler", "babbar") 392 a("SurdotlyBot", "surdotly") 393 a("DomCopBot", "domcop") 394 a("NetcraftSurveyAgent", "netcraft") 395 a("seostar", "seostar") 396 a("Pandalytics", "pandalytics") 397 a("BLEXBot", "blexbot") 398 a("Wappalyzer", "wappalyzer") 399 a("Twingly", "twingly") 400 a("linkfluence", "linkfluence") 401 a("PageThing", "pagething") 402 a("Burf", "burf") 403 a("AspiegelBot", "aspiegel") 404 a("PetalBot", "aspiegel") 405 a("RU_Bot", "mail_ru") // Mail.RU_Bot 406 a("CCBot", "ccbot") 407 a("yacybot", "yacy") 408 a("zgrab", "zgrab") 409 a("Nimbostratus", "cloudsystemnetworks") 410 a("MauiBot", "maui") 411 a("curl", "curl") 412 a("python", "python") 413 //a("Go", "go") // yacy has java as part of it's UA, try to avoid hitting crawlers written in go 414 a("HeadlessChrome", "headlesschrome") 415 a("awesome_bot", "awesome_bot") 416 // TODO: Detect Adsbot/3.1, it has a similar user agent to Google's Adsbot, but it is different. No Google fragments. 417 418 tmplVars.AllAgentMarkIDs = make(map[string]int) 419 for mark, agent := range tmplVars.AllAgentMarks { 420 tmplVars.AllAgentMarkIDs[mark] = tmplVars.AllAgentMap[agent] 421 } 422 423 fileData := `// Code generated by Gosora's Router Generator. DO NOT EDIT. 424 /* This file was automatically generated by the software. Please don't edit it as your changes may be overwritten at any moment. */ 425 package main 426 427 import ( 428 "strings" 429 //"bytes" 430 "strconv" 431 "compress/gzip" 432 "sync/atomic" 433 "errors" 434 "net/http" 435 436 c "github.com/Azareal/Gosora/common" 437 co "github.com/Azareal/Gosora/common/counters" 438 "github.com/Azareal/Gosora/uutils" 439 "github.com/Azareal/Gosora/routes" 440 "github.com/Azareal/Gosora/routes/panel" 441 442 //"github.com/andybalholm/brotli" 443 ) 444 445 var ErrNoRoute = errors.New("That route doesn't exist.") 446 // TODO: What about the /uploads/ route? x.x 447 var RouteMap = map[string]interface{}{ {{range .AllRouteNames}} 448 "{{.Plain}}": {{.Short}},{{end}} 449 } 450 451 // ! NEVER RELY ON THESE REMAINING THE SAME BETWEEN COMMITS 452 var routeMapEnum = map[string]int{ {{range $index, $el := .AllRouteNames}} 453 "{{$el.Plain}}": {{$index}},{{end}} 454 } 455 var reverseRouteMapEnum = map[int]string{ {{range $index, $el := .AllRouteNames}} 456 {{$index}}: "{{$el.Plain}}",{{end}} 457 } 458 var osMapEnum = map[string]int{ {{range $index, $el := .AllOSNames}} 459 "{{$el}}": {{$index}},{{end}} 460 } 461 var reverseOSMapEnum = map[int]string{ {{range $index, $el := .AllOSNames}} 462 {{$index}}: "{{$el}}",{{end}} 463 } 464 var agentMapEnum = map[string]int{ {{range $index, $el := .AllAgentNames}} 465 "{{$el}}": {{$index}},{{end}} 466 } 467 var reverseAgentMapEnum = map[int]string{ {{range $index, $el := .AllAgentNames}} 468 {{$index}}: "{{$el}}",{{end}} 469 } 470 var markToAgent = map[string]string{ {{range $index, $el := .AllAgentMarkNames}} 471 "{{$el}}": "{{index $.AllAgentMarks $el}}",{{end}} 472 } 473 var markToID = map[string]int{ {{range $index, $el := .AllAgentMarkNames}} 474 "{{$el}}": {{index $.AllAgentMarkIDs $el}},{{end}} 475 } 476 /*var agentRank = map[string]int{ 477 "opera":9, 478 "chrome":8, 479 "safari":1, 480 }*/ 481 482 // HTTPSRedirect is a connection handler which redirects all HTTP requests to HTTPS 483 type HTTPSRedirect struct {} 484 485 func (red *HTTPSRedirect) ServeHTTP(w http.ResponseWriter, req *http.Request) { 486 w.Header().Set("Connection", "close") 487 co.RouteViewCounter.Bump({{index .AllRouteMap "routes.HTTPSRedirect"}}) 488 dest := "https://" + req.Host + req.URL.String() 489 http.Redirect(w, req, dest, http.StatusTemporaryRedirect) 490 } 491 492 func (r *GenRouter) SuspiciousRequest(req *http.Request, pre string) { 493 if c.Config.DisableSuspLog { 494 return 495 } 496 var sb strings.Builder 497 if pre != "" { 498 sb.WriteString("Suspicious Request\n") 499 } else { 500 pre = "Suspicious Request" 501 } 502 r.ddumpRequest(req,pre,r.suspLog,&sb) 503 co.AgentViewCounter.Bump({{.AllAgentMap.suspicious}}) 504 } 505 506 // TODO: Pass the default path or config struct to the router rather than accessing it via a package global 507 // TODO: SetDefaultPath 508 // TODO: GetDefaultPath 509 func (r *GenRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { 510 malformedRequest := func(typ int) { 511 w.WriteHeader(200) // 400 512 w.Write([]byte("")) 513 r.DumpRequest(req,"Malformed Request T"+strconv.Itoa(typ)) 514 co.AgentViewCounter.Bump({{.AllAgentMap.malformed}}) 515 } 516 517 // Split the Host and Port string 518 var shost, sport string 519 if req.Host[0]=='[' { 520 spl := strings.Split(req.Host,"]") 521 if len(spl) > 2 { 522 malformedRequest(0) 523 return 524 } 525 shost = strings.TrimPrefix(spl[0],"[") 526 sport = strings.TrimPrefix(spl[1],":") 527 } else if strings.Contains(req.Host,":") { 528 spl := strings.Split(req.Host,":") 529 if len(spl) > 2 { 530 malformedRequest(1) 531 return 532 } 533 shost = spl[0] 534 //if len(spl)==2 { 535 sport = spl[1] 536 //} 537 } else { 538 shost = req.Host 539 } 540 // TODO: Reject requests from non-local IPs, if the site host is set to localhost or a localhost IP 541 if !c.Config.LoosePort && c.Site.PortInt != 80 && c.Site.PortInt != 443 && sport != c.Site.Port { 542 malformedRequest(2) 543 return 544 } 545 546 // Redirect www. and local IP requests to the right place 547 if strings.HasPrefix(shost, "www.") || c.Site.LocalHost { 548 if shost == "www." + c.Site.Host || (c.Site.LocalHost && shost != c.Site.Host && isLocalHost(shost)) { 549 // TODO: Abstract the redirect logic? 550 w.Header().Set("Connection", "close") 551 var s, p string 552 if c.Config.SslSchema { 553 s = "s" 554 } 555 if c.Site.PortInt != 80 && c.Site.PortInt != 443 { 556 p = ":"+c.Site.Port 557 } 558 dest := "http"+s+"://" + c.Site.Host+p + req.URL.Path 559 if len(req.URL.RawQuery) > 0 { 560 dest += "?" + req.URL.RawQuery 561 } 562 http.Redirect(w, req, dest, http.StatusMovedPermanently) 563 return 564 } 565 } 566 567 // Deflect malformed requests 568 if len(req.URL.Path) == 0 || req.URL.Path[0] != '/' || (!c.Config.LooseHost && shost != c.Site.Host) { 569 malformedRequest(3) 570 return 571 } 572 r.suspScan(req) 573 574 // Indirect the default route onto a different one 575 if req.URL.Path == "/" { 576 req.URL.Path = c.Config.DefaultPath 577 } 578 //log.Print("URL.Path: ", req.URL.Path) 579 prefix := req.URL.Path[0:strings.IndexByte(req.URL.Path[1:],'/') + 1] 580 581 // TODO: Use the same hook table as downstream 582 hTbl := c.GetHookTable() 583 skip, ferr := c.H_router_after_filters_hook(hTbl, w, req, prefix) 584 if skip || ferr != nil { 585 return 586 } 587 588 if prefix != "/ws" { 589 h := w.Header() 590 h.Set("X-Frame-Options", "deny") 591 h.Set("X-XSS-Protection", "1; mode=block") // TODO: Remove when we add a CSP? CSP's are horrendously glitchy things, tread with caution before removing 592 h.Set("X-Content-Type-Options", "nosniff") 593 if c.Config.RefNoRef || !c.Config.SslSchema { 594 h.Set("Referrer-Policy","no-referrer") 595 } else { 596 h.Set("Referrer-Policy","strict-origin") 597 } 598 h.Set("Permissions-Policy","interest-cohort=()") 599 } 600 601 if c.Dev.SuperDebug { 602 r.DumpRequest(req,"before routes.StaticFile") 603 } 604 // Increment the request counter 605 if !c.Config.DisableAnalytics { 606 co.GlobalViewCounter.Bump() 607 } 608 609 if prefix == "/s" { //old prefix: /static 610 if !c.Config.DisableAnalytics { 611 co.RouteViewCounter.Bump({{index .AllRouteMap "routes.StaticFile"}}) 612 } 613 routes.StaticFile(w, req) 614 return 615 } 616 // TODO: Handle JS routes 617 if atomic.LoadInt32(&c.IsDBDown) == 1 { 618 c.DatabaseError(w, req) 619 return 620 } 621 if c.Dev.SuperDebug { 622 r.reqLogger.Print("before PreRoute") 623 } 624 625 /*if c.Dev.QuicPort != 0 { 626 sQuicPort := strconv.Itoa(c.Dev.QuicPort) 627 w.Header().Set("Alt-Svc", "quic=\":"+sQuicPort+"\"; ma=2592000; v=\"44,43,39\", h3-23=\":"+sQuicPort+"\"; ma=3600, h3-24=\":"+sQuicPort+"\"; ma=3600, h2=\":443\"; ma=3600") 628 }*/ 629 630 // Track the user agents. Unfortunately, everyone pretends to be Mozilla, so this'll be a little less efficient than I would like. 631 // TODO: Add a setting to disable this? 632 // TODO: Use a more efficient detector instead of smashing every possible combination in 633 // TODO: Make this testable 634 var agent int 635 if !c.Config.DisableAnalytics { 636 637 ua := strings.TrimSpace(strings.Replace(strings.TrimPrefix(req.UserAgent(),"Mozilla/5.0 ")," Safari/537.36","",-1)) // Noise, no one's going to be running this and it would require some sort of agent ranking system to determine which identifier should be prioritised over another 638 if ua == "" { 639 co.AgentViewCounter.Bump({{.AllAgentMap.blank}}) 640 r.unknownUA(req) 641 } else { 642 // WIP UA Parser 643 //var ii = uaBufPool.Get() 644 var buf []byte 645 //if ii != nil { 646 // buf = ii.([]byte) 647 //} 648 var items []string 649 var os int 650 for _, it := range uutils.StringToBytes(ua) { 651 if (it > 64 && it < 91) || (it > 96 && it < 123) || (it > 47 && it < 58) || it == '_' { 652 // TODO: Store an index and slice that instead? 653 buf = append(buf, it) 654 } else if it == ' ' || it == '(' || it == ')' || it == '-' || it == ';' || it == ':' || it == '.' || it == '+' || it == '~' || it == '@' /*|| (it == ':' && bytes.Equal(buf,[]byte("http")))*/ || it == ',' || it == '/' { 655 //log.Print("buf: ",string(buf)) 656 //log.Print("it: ",string(it)) 657 if len(buf) != 0 { 658 if len(buf) > 2 { 659 // Use an unsafe zero copy conversion here just to use the switch, it's not safe for this string to escape from here, as it will get mutated, so do a regular string conversion in append 660 switch(uutils.BytesToString(buf)) { 661 case "Windows": 662 os = {{.AllOSMap.windows}} 663 case "Linux": 664 os = {{.AllOSMap.linux}} 665 case "Mac": 666 os = {{.AllOSMap.mac}} 667 case "iPhone": 668 os = {{.AllOSMap.iphone}} 669 case "Android": 670 os = {{.AllOSMap.android}} 671 case "like","compatible","NT","X","com","KHTML": 672 // Skip these words 673 default: 674 //log.Print("append buf") 675 items = append(items, string(buf)) 676 } 677 } 678 //log.Print("reset buf") 679 buf = buf[:0] 680 } 681 } else { 682 // TODO: Test this 683 items = items[:0] 684 if c.Config.DisableSuspLog { 685 r.reqLogger.Print("Illegal char "+strconv.Itoa(int(it))+" in UA\nUA Buf: ", buf,"\nUA Buf String: ", string(buf)) 686 } else { 687 r.SuspiciousRequest(req,"Illegal char "+strconv.Itoa(int(it))+" in UA") 688 r.reqLogger.Print("UA Buf: ", buf,"\nUA Buf String: ", string(buf)) 689 } 690 break 691 } 692 } 693 //uaBufPool.Put(buf) 694 695 // Iterate over this in reverse as the real UA tends to be on the right side 696 for i := len(items) - 1; i >= 0; i-- { 697 //fAgent, ok := markToAgent[items[i]] 698 fAgent, ok := markToID[items[i]] 699 if ok { 700 agent = fAgent 701 if agent != {{.AllAgentMap.safari}} { 702 break 703 } 704 } 705 } 706 if c.Dev.SuperDebug { 707 r.reqLogger.Print("parsed agent: ", agent,"\nos: ", os) 708 r.reqLogger.Printf("items: %+v\n",items) 709 /*for _, it := range items { 710 r.reqLogger.Printf("it: %+v\n",string(it)) 711 }*/ 712 } 713 714 // Special handling 715 switch(agent) { 716 case {{.AllAgentMap.chrome}}: 717 if os == {{.AllOSMap.android}} { 718 agent = {{.AllAgentMap.androidchrome}} 719 } 720 case {{.AllAgentMap.safari}}: 721 if os == {{.AllOSMap.iphone}} { 722 agent = {{.AllAgentMap.mobilesafari}} 723 } 724 case {{.AllAgentMap.trident}}: 725 // Hack to support IE11, change this after we start logging versions 726 if strings.Contains(ua,"rv:11") { 727 agent = {{.AllAgentMap.internetexplorer}} 728 } 729 case {{.AllAgentMap.zgrab}}: 730 w.WriteHeader(200) // 400 731 w.Write([]byte("")) 732 r.DumpRequest(req,"Blocked Scanner") 733 co.AgentViewCounter.Bump({{.AllAgentMap.zgrab}}) 734 return 735 } 736 737 if agent == 0 { 738 //co.AgentViewCounter.Bump({{.AllAgentMap.unknown}}) 739 r.unknownUA(req) 740 }// else { 741 //co.AgentViewCounter.Bump(agentMapEnum[agent]) 742 co.AgentViewCounter.Bump(agent) 743 //} 744 co.OSViewCounter.Bump(os) 745 } 746 747 // TODO: Do we want to track missing language headers too? Maybe as it's own type, e.g. "noheader"? 748 // TODO: Default to anything other than en, if anything else is present, to avoid over-representing it for multi-linguals? 749 lang := req.Header.Get("Accept-Language") 750 if lang != "" { 751 // TODO: Reduce allocs here 752 lLang := strings.Split(strings.TrimSpace(lang),"-") 753 tLang := strings.Split(strings.Split(lLang[0],";")[0],",") 754 c.DebugDetail("tLang:", tLang) 755 var llLang string 756 for _, seg := range tLang { 757 if seg == "*" { 758 continue 759 } 760 llLang = seg 761 break 762 } 763 c.DebugDetail("llLang:", llLang) 764 if !co.LangViewCounter.Bump(llLang) { 765 r.DumpRequest(req,"Invalid ISO Code") 766 } 767 } else { 768 co.LangViewCounter.Bump2(0) 769 } 770 771 if !c.Config.RefNoTrack { 772 ae := req.Header.Get("Accept-Encoding") 773 likelyBot := ae == "gzip" || ae == "" 774 if !likelyBot { 775 ref := req.Header.Get("Referer") // Check the 'referrer' header too? :P 776 // TODO: Extend the effects of DNT elsewhere? 777 if ref != "" && req.Header.Get("DNT") != "1" { 778 // ? Optimise this a little? 779 ref = strings.TrimPrefix(strings.TrimPrefix(ref,"http://"),"https://") 780 ref = strings.Split(ref,"/")[0] 781 portless := strings.Split(ref,":")[0] 782 // TODO: Handle c.Site.Host in uppercase too? 783 if portless != "localhost" && portless != "127.0.0.1" && portless != c.Site.Host { 784 r.DumpRequest(req,"Ref Route") 785 co.ReferrerTracker.Bump(ref) 786 } 787 } 788 } 789 } 790 791 } 792 793 // Deal with the session stuff, etc. 794 ucpy, ok := c.PreRoute(w, req) 795 if !ok { 796 return 797 } 798 user := &ucpy 799 user.LastAgent = agent 800 if c.Dev.SuperDebug { 801 r.reqLogger.Print( 802 "after PreRoute\n" + 803 "routeMapEnum: ", routeMapEnum) 804 } 805 //log.Println("req: ", req) 806 807 // Disable Gzip when SSL is disabled for security reasons? 808 if prefix != "/ws" { 809 ae := req.Header.Get("Accept-Encoding") 810 /*if strings.Contains(ae, "br") { 811 h := w.Header() 812 h.Set("Content-Encoding", "br") 813 var ii = brPool.Get() 814 var igzw *brotli.Writer 815 if ii == nil { 816 igzw = brotli.NewWriter(w) 817 } else { 818 igzw = ii.(*brotli.Writer) 819 igzw.Reset(w) 820 } 821 gzw := c.BrResponseWriter{Writer: igzw, ResponseWriter: w} 822 defer func() { 823 //h := w.Header() 824 if h.Get("Content-Encoding") == "br" && h.Get("X-I") == "" { 825 //log.Print("push br close") 826 igzw := gzw.Writer.(*brotli.Writer) 827 igzw.Close() 828 brPool.Put(igzw) 829 } 830 }() 831 w = gzw 832 } else */if strings.Contains(ae, "gzip") { 833 h := w.Header() 834 h.Set("Content-Encoding", "gzip") 835 var ii = gzipPool.Get() 836 var igzw *gzip.Writer 837 if ii == nil { 838 igzw = gzip.NewWriter(w) 839 } else { 840 igzw = ii.(*gzip.Writer) 841 igzw.Reset(w) 842 } 843 gzw := c.GzipResponseWriter{Writer: igzw, ResponseWriter: w} 844 defer func() { 845 //h := w.Header() 846 if h.Get("Content-Encoding") == "gzip" && h.Get("X-I") == "" { 847 //log.Print("push gzip close") 848 igzw := gzw.Writer.(*gzip.Writer) 849 igzw.Close() 850 gzipPool.Put(igzw) 851 } 852 }() 853 w = gzw 854 } 855 } 856 857 skip, ferr = c.H_router_pre_route_hook(hTbl, w, req, user, prefix) 858 if skip || ferr != nil { 859 r.handleError(ferr,w,req,user) 860 return 861 } 862 var extraData string 863 if req.URL.Path[len(req.URL.Path) - 1] != '/' { 864 extraData = req.URL.Path[strings.LastIndexByte(req.URL.Path,'/') + 1:] 865 req.URL.Path = req.URL.Path[:strings.LastIndexByte(req.URL.Path,'/') + 1] 866 } 867 ferr = r.routeSwitch(w, req, user, prefix, extraData) 868 if ferr != nil { 869 r.handleError(ferr,w,req,user) 870 return 871 } 872 /*if !c.Config.DisableAnalytics { 873 co.RouteViewCounter.Bump(id) 874 }*/ 875 876 hTbl.VhookNoRet("router_end", w, req, user, prefix, extraData) 877 //c.StoppedServer("Profile end") 878 } 879 880 func (r *GenRouter) routeSwitch(w http.ResponseWriter, req *http.Request, user *c.User, prefix, extraData string) /*(id int, orerr */c.RouteError/*)*/ { 881 var err c.RouteError 882 cn := uutils.Nanotime() 883 switch(prefix) {` + out + ` 884 /*case "/sitemaps": // TODO: Count these views 885 req.URL.Path += extraData 886 err = sitemapSwitch(w,req)*/ 887 // ! Temporary fix for certain bots 888 case "/static": 889 w.Header().Set("Connection", "close") 890 http.Redirect(w, req, "/s/"+extraData, http.StatusTemporaryRedirect) 891 case "/uploads": 892 if extraData == "" { 893 co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.UploadedFile"}}, cn) 894 return c.NotFound(w,req,nil) 895 } 896 w = r.responseWriter(w) 897 req.URL.Path += extraData 898 // TODO: Find a way to propagate errors up from this? 899 r.UploadHandler(w,req) // TODO: Count these views 900 co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.UploadedFile"}}, cn) 901 return nil 902 case "": 903 // Stop the favicons, robots.txt file, etc. resolving to the topics list 904 // TODO: Add support for favicons and robots.txt files 905 switch(extraData) { 906 case "robots.txt": 907 co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.RobotsTxt"}}, cn) 908 return routes.RobotsTxt(w,req) 909 case "favicon.ico": 910 w = r.responseWriter(w) 911 req.URL.Path = "/s/favicon.ico" 912 co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.Favicon"}}, cn) 913 routes.StaticFile(w,req) 914 return nil 915 case "opensearch.xml": 916 co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.OpenSearchXml"}}, cn) 917 return routes.OpenSearchXml(w,req) 918 /*case "sitemap.xml": 919 co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.SitemapXml"}}, cn) 920 return routes.SitemapXml(w,req)*/ 921 } 922 co.RouteViewCounter.Bump({{index .AllRouteMap "routes.Error"}}) 923 return c.NotFound(w,req,nil) 924 default: 925 // A fallback for dynamic routes, e.g. ones declared by plugins 926 r.RLock() 927 h, ok := r.extraRoutes[req.URL.Path] 928 r.RUnlock() 929 req.URL.Path += extraData 930 931 if ok { 932 // TODO: Be more specific about *which* dynamic route it is 933 co.RouteViewCounter.Bump({{index .AllRouteMap "routes.DynamicRoute"}}) 934 return h(w,req,user) 935 } 936 co.RouteViewCounter.Bump3({{index .AllRouteMap "routes.BadRoute"}}, cn) 937 938 if !c.Config.DisableSuspLog { 939 lp := strings.ToLower(req.URL.Path) 940 if strings.Contains(lp,"w") { 941 if strings.Contains(lp,"wp") || strings.Contains(lp,"wordpress") || strings.Contains(lp,"wget") || strings.Contains(lp,"wp-") { 942 r.SuspiciousRequest(req,"Bad Route") 943 return c.MicroNotFound(w,req) 944 } 945 } 946 if strings.Contains(lp,"admin") || strings.Contains(lp,"sql") || strings.Contains(lp,"manage") || strings.Contains(lp,"//") || strings.Contains(lp,"\\\\") || strings.Contains(lp,"config") || strings.Contains(lp,"setup") || strings.Contains(lp,"install") || strings.Contains(lp,"update") || strings.Contains(lp,"php") || strings.Contains(lp,"pl") || strings.Contains(lp,"include") || strings.Contains(lp,"vendor") || strings.Contains(lp,"bin") || strings.Contains(lp,"system") || strings.Contains(lp,"eval") || strings.Contains(lp,"config") { 947 r.SuspiciousRequest(req,"Bad Route") 948 return c.MicroNotFound(w,req) 949 } 950 } 951 952 if !c.Config.DisableBadRouteLog { 953 r.DumpRequest(req,"Bad Route") 954 } 955 ae := req.Header.Get("Accept-Encoding") 956 likelyBot := ae == "gzip" || ae == "" 957 if likelyBot { 958 return c.MicroNotFound(w,req) 959 } 960 return c.NotFound(w,req,nil) 961 } 962 return err 963 } 964 ` 965 tmpl := template.Must(template.New("router").Parse(fileData)) 966 var b bytes.Buffer 967 if err := tmpl.Execute(&b, tmplVars); err != nil { 968 log.Fatal(err) 969 } 970 971 writeFile("./gen_router.go", b.String()) 972 log.Println("Successfully generated the router") 973 } 974 975 func writeFile(name, content string) { 976 f, e := os.Create(name) 977 if e != nil { 978 log.Fatal(e) 979 } 980 _, e = f.WriteString(content) 981 if e != nil { 982 log.Fatal(e) 983 } 984 if e = f.Sync(); e != nil { 985 log.Fatal(e) 986 } 987 if e = f.Close(); e != nil { 988 log.Fatal(e) 989 } 990 }