github.com/Azareal/Gosora@v0.0.0-20210729070923-553e66b59003/router.go (about)

     1  // Now home to the parts of gen_router.go which aren't expected to change from generation to generation
     2  package main
     3  
     4  import (
     5  	"log"
     6  	"net/http"
     7  	"os"
     8  	"strconv"
     9  	"strings"
    10  	"sync"
    11  	"sync/atomic"
    12  	"time"
    13  
    14  	c "github.com/Azareal/Gosora/common"
    15  	co "github.com/Azareal/Gosora/common/counters"
    16  )
    17  
    18  // TODO: Stop spilling these into the package scope?
    19  func init() {
    20  	_ = time.Now()
    21  	co.SetRouteMapEnum(routeMapEnum)
    22  	co.SetReverseRouteMapEnum(reverseRouteMapEnum)
    23  	co.SetAgentMapEnum(agentMapEnum)
    24  	co.SetReverseAgentMapEnum(reverseAgentMapEnum)
    25  	co.SetOSMapEnum(osMapEnum)
    26  	co.SetReverseOSMapEnum(reverseOSMapEnum)
    27  
    28  	g := func(n string) int {
    29  		a, ok := agentMapEnum[n]
    30  		if !ok {
    31  			panic("name not found in agentMapEnum")
    32  		}
    33  		return a
    34  	}
    35  	c.Chrome = g("chrome")
    36  	c.Firefox = g("firefox")
    37  	c.SimpleBots = []int{
    38  		g("semrush"),
    39  		g("ahrefs"),
    40  		g("python"),
    41  		//g("go"),
    42  		g("curl"),
    43  	}
    44  }
    45  
    46  type WriterIntercept struct {
    47  	http.ResponseWriter
    48  }
    49  
    50  func NewWriterIntercept(w http.ResponseWriter) *WriterIntercept {
    51  	return &WriterIntercept{w}
    52  }
    53  
    54  var wiMaxAge = "max-age=" + strconv.Itoa(int(c.Day))
    55  
    56  func (wi *WriterIntercept) WriteHeader(code int) {
    57  	if code == 200 {
    58  		h := wi.ResponseWriter.Header()
    59  		h.Set("Cache-Control", wiMaxAge)
    60  		h.Set("Vary", "Accept-Encoding")
    61  	}
    62  	wi.ResponseWriter.WriteHeader(code)
    63  }
    64  
    65  type GenRouter struct {
    66  	UploadHandler func(http.ResponseWriter, *http.Request)
    67  	extraRoutes   map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError
    68  
    69  	reqLogger *log.Logger
    70  
    71  	reqLog2 *RouterLog
    72  	suspLog *RouterLog
    73  
    74  	sync.RWMutex
    75  }
    76  
    77  type RouterLogLog struct {
    78  	File *os.File
    79  	Log  *log.Logger
    80  }
    81  type RouterLog struct {
    82  	FileVal atomic.Value
    83  	LogVal  atomic.Value
    84  
    85  	sync.RWMutex
    86  }
    87  
    88  func (r *GenRouter) DailyTick() error {
    89  	currentTime := time.Now()
    90  	rotateLog := func(l *RouterLog, name string) error {
    91  		l.Lock()
    92  		defer l.Unlock()
    93  
    94  		f := l.FileVal.Load().(*os.File)
    95  		stat, e := f.Stat()
    96  		if e != nil {
    97  			return nil
    98  		}
    99  		if (stat.Size() < int64(c.Megabyte)) && (currentTime.Sub(c.StartTime).Hours() >= (24 * 7)) {
   100  			return nil
   101  		}
   102  		if e = f.Close(); e != nil {
   103  			return e
   104  		}
   105  
   106  		stimestr := strconv.FormatInt(currentTime.Unix(), 10)
   107  		f, e = os.OpenFile(c.Config.LogDir+name+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
   108  		if e != nil {
   109  			return e
   110  		}
   111  		lval := log.New(f, "", log.LstdFlags)
   112  		l.FileVal.Store(f)
   113  		l.LogVal.Store(lval)
   114  		return nil
   115  	}
   116  
   117  	if !c.Config.DisableSuspLog {
   118  		err := rotateLog(r.suspLog, "reqs-susp-")
   119  		if err != nil {
   120  			return err
   121  		}
   122  	}
   123  	return rotateLog(r.reqLog2, "reqs-")
   124  }
   125  
   126  type RouterConfig struct {
   127  	Uploads     http.Handler
   128  	DisableTick bool
   129  }
   130  
   131  func NewGenRouter(cfg *RouterConfig) (*GenRouter, error) {
   132  	stimestr := strconv.FormatInt(c.StartTime.Unix(), 10)
   133  	createLog := func(name, stimestr string) (*RouterLog, error) {
   134  		f, err := os.OpenFile(c.Config.LogDir+name+"-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
   135  		if err != nil {
   136  			return nil, err
   137  		}
   138  		l := log.New(f, "", log.LstdFlags)
   139  		var aVal atomic.Value
   140  		var aVal2 atomic.Value
   141  		aVal.Store(f)
   142  		aVal2.Store(l)
   143  		return &RouterLog{FileVal: aVal, LogVal: aVal2}, nil
   144  	}
   145  	reqLog, err := createLog("reqs", stimestr)
   146  	if err != nil {
   147  		return nil, err
   148  	}
   149  	var suspReqLog *RouterLog
   150  	if !c.Config.DisableSuspLog {
   151  		suspReqLog, err = createLog("reqs-susp", stimestr)
   152  		if err != nil {
   153  			return nil, err
   154  		}
   155  	}
   156  	f3, err := os.OpenFile(c.Config.LogDir+"reqs-misc-"+stimestr+".log", os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0755)
   157  	if err != nil {
   158  		return nil, err
   159  	}
   160  	reqMiscLog := log.New(f3, "", log.LstdFlags)
   161  
   162  	ro := &GenRouter{
   163  		UploadHandler: func(w http.ResponseWriter, r *http.Request) {
   164  			writ := NewWriterIntercept(w)
   165  			http.StripPrefix("/uploads/", cfg.Uploads).ServeHTTP(writ, r)
   166  		},
   167  		extraRoutes: make(map[string]func(http.ResponseWriter, *http.Request, *c.User) c.RouteError),
   168  
   169  		reqLogger: reqMiscLog,
   170  		reqLog2:   reqLog,
   171  		suspLog:   suspReqLog,
   172  	}
   173  	if !cfg.DisableTick {
   174  		c.Tasks.Day.Add(ro.DailyTick)
   175  	}
   176  	return ro, nil
   177  }
   178  
   179  func (r *GenRouter) handleError(err c.RouteError, w http.ResponseWriter, req *http.Request, u *c.User) {
   180  	if err.Handled() {
   181  		return
   182  	}
   183  	if err.Type() == "system" {
   184  		c.InternalErrorJSQ(err, w, req, err.JSON())
   185  		return
   186  	}
   187  	c.LocalErrorJSQ(err.Error(), w, req, u, err.JSON())
   188  }
   189  
   190  func (r *GenRouter) Handle(_ string, _ http.Handler) {
   191  }
   192  
   193  func (r *GenRouter) HandleFunc(pattern string, h func(http.ResponseWriter, *http.Request, *c.User) c.RouteError) {
   194  	r.Lock()
   195  	defer r.Unlock()
   196  	r.extraRoutes[pattern] = h
   197  }
   198  
   199  func (r *GenRouter) RemoveFunc(pattern string) error {
   200  	r.Lock()
   201  	defer r.Unlock()
   202  	_, ok := r.extraRoutes[pattern]
   203  	if !ok {
   204  		return ErrNoRoute
   205  	}
   206  	delete(r.extraRoutes, pattern)
   207  	return nil
   208  }
   209  
   210  func (r *GenRouter) dumpRequest(req *http.Request, pre string, log *RouterLog) {
   211  	var sb strings.Builder
   212  	r.ddumpRequest(req, pre, log, &sb)
   213  }
   214  
   215  // TODO: Some of these sanitisations may be redundant
   216  var dumpReqLen = len("\nUA: \n Host: \nIP: \n") + 7
   217  var dumpReqLen2 = len("\nHead : ") + 2
   218  
   219  func (r *GenRouter) ddumpRequest(req *http.Request, pre string, l *RouterLog, sb *strings.Builder) {
   220  	nfield := func(label, val string) {
   221  		sb.WriteString(label)
   222  		sb.WriteString(val)
   223  	}
   224  	field := func(label, val string) {
   225  		nfield(label, c.SanitiseSingleLine(val))
   226  	}
   227  	ua := req.UserAgent()
   228  
   229  	sb.Grow(dumpReqLen + len(pre) + len(ua) + len(req.Method) + len(req.Host) + (dumpReqLen2 * len(req.Header)))
   230  	sb.WriteString(pre)
   231  	sb.WriteString("\n")
   232  	sb.WriteString(c.SanitiseSingleLine(req.Method))
   233  	sb.WriteRune(' ')
   234  	sb.WriteString(c.SanitiseSingleLine(req.URL.Path))
   235  	field("\nUA: ", ua)
   236  
   237  	for key, val := range req.Header {
   238  		// Avoid logging this for security reasons
   239  		if key == "Cookie" {
   240  			continue
   241  		}
   242  		for _, vvalue := range val {
   243  			sb.WriteString("\nHead ")
   244  			sb.WriteString(c.SanitiseSingleLine(key))
   245  			sb.WriteString(": ")
   246  			sb.WriteString(c.SanitiseSingleLine(vvalue))
   247  		}
   248  	}
   249  	field("\nHost: ", req.Host)
   250  	if rawQuery := req.URL.RawQuery; rawQuery != "" {
   251  		field("\nURL.RawQuery: ", rawQuery)
   252  	}
   253  	if ref := req.Referer(); ref != "" {
   254  		field("\nRef: ", ref)
   255  	}
   256  	nfield("\nIP: ", req.RemoteAddr)
   257  	sb.WriteString("\n")
   258  
   259  	str := sb.String()
   260  	l.RLock()
   261  	l.LogVal.Load().(*log.Logger).Print(str)
   262  	l.RUnlock()
   263  }
   264  
   265  func (r *GenRouter) DumpRequest(req *http.Request, pre string) {
   266  	r.dumpRequest(req, pre, r.reqLog2)
   267  }
   268  
   269  func (r *GenRouter) unknownUA(req *http.Request) {
   270  	if c.Dev.DebugMode {
   271  		var presb strings.Builder
   272  		presb.WriteString("Unknown UA: ")
   273  		for _, ch := range req.UserAgent() {
   274  			presb.WriteString(strconv.Itoa(int(ch)))
   275  			presb.WriteRune(' ')
   276  		}
   277  		r.ddumpRequest(req, "", r.reqLog2, &presb)
   278  	} else {
   279  		r.reqLogger.Print("unknown ua: ", c.SanitiseSingleLine(req.UserAgent()))
   280  	}
   281  }
   282  
   283  func (r *GenRouter) susp1(req *http.Request) bool {
   284  	if !strings.Contains(req.URL.Path, ".") {
   285  		return false
   286  	}
   287  	if strings.Contains(req.URL.Path, "..") /* || strings.Contains(req.URL.Path,"--")*/ {
   288  		return true
   289  	}
   290  	lp := strings.ToLower(req.URL.Path)
   291  	// TODO: Flag any requests which has a dot with anything but a number after that
   292  	// TODO: Use HasSuffix to avoid over-scanning?
   293  	return strings.Contains(lp, ".php") || strings.Contains(lp, ".asp") || strings.Contains(lp, ".cgi") || strings.Contains(lp, ".py") || strings.Contains(lp, ".sql") || strings.Contains(lp, ".act") //.action
   294  }
   295  
   296  func (r *GenRouter) suspScan(req *http.Request) {
   297  	if c.Config.DisableSuspLog {
   298  		if c.Dev.FullReqLog {
   299  			r.DumpRequest(req, "")
   300  		}
   301  		return
   302  	}
   303  
   304  	// TODO: Cover more suspicious strings and at a lower layer than this
   305  	var ch rune
   306  	var susp bool
   307  	for _, ch = range req.URL.Path { //char
   308  		if ch != '&' && !(ch > 44 && ch < 58) && ch != '=' && ch != '?' && !(ch > 64 && ch < 91) && ch != '\\' && ch != '_' && !(ch > 96 && ch < 123) {
   309  			susp = true
   310  			break
   311  		}
   312  	}
   313  
   314  	// Avoid logging the same request multiple times
   315  	susp2 := r.susp1(req)
   316  	if susp && susp2 {
   317  		r.SuspiciousRequest(req, "Bad char '"+string(ch)+"' in path\nBad snippet in path")
   318  	} else if susp {
   319  		r.SuspiciousRequest(req, "Bad char '"+string(ch)+"' in path")
   320  	} else if susp2 {
   321  		r.SuspiciousRequest(req, "Bad snippet in path")
   322  	} else if c.Dev.FullReqLog {
   323  		r.DumpRequest(req, "")
   324  	}
   325  }
   326  
   327  func isLocalHost(h string) bool {
   328  	return h == "localhost" || h == "127.0.0.1" || h == "::1"
   329  }
   330  
   331  //var brPool = sync.Pool{}
   332  var gzipPool = sync.Pool{}
   333  
   334  //var uaBufPool = sync.Pool{}
   335  
   336  func (r *GenRouter) responseWriter(w http.ResponseWriter) http.ResponseWriter {
   337  	/*if bzw, ok := w.(c.BrResponseWriter); ok {
   338  		w = bzw.ResponseWriter
   339  		w.Header().Del("Content-Encoding")
   340  	} else */if gzw, ok := w.(c.GzipResponseWriter); ok {
   341  		w = gzw.ResponseWriter
   342  		w.Header().Del("Content-Encoding")
   343  	}
   344  	return w
   345  }