github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/website/camweb.go (about) 1 /* 2 Copyright 2011 Google Inc. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package main 18 19 import ( 20 "bytes" 21 "flag" 22 "fmt" 23 "html/template" 24 "io" 25 "io/ioutil" 26 "log" 27 "net/http" 28 "net/http/httputil" 29 "net/url" 30 "os" 31 "os/exec" 32 "path/filepath" 33 "regexp" 34 "strings" 35 txttemplate "text/template" 36 "time" 37 ) 38 39 const defaultAddr = ":31798" // default webserver address 40 41 var h1TitlePattern = regexp.MustCompile(`<h1>([^<]+)</h1>`) 42 43 var ( 44 httpAddr = flag.String("http", defaultAddr, "HTTP service address (e.g., '"+defaultAddr+"')") 45 httpsAddr = flag.String("https", "", "HTTPS service address") 46 root = flag.String("root", "", "Website root (parent of 'static', 'content', and 'tmpl") 47 logDir = flag.String("logdir", "", "Directory to write log files to (one per hour), or empty to not log.") 48 logStdout = flag.Bool("logstdout", true, "Write to stdout?") 49 tlsCertFile = flag.String("tlscert", "", "TLS cert file") 50 tlsKeyFile = flag.String("tlskey", "", "TLS private key file") 51 buildbotBackend = flag.String("buildbot_backend", "", "Build bot status backend URL") 52 buildbotHost = flag.String("buildbot_host", "", "Hostname to map to the buildbot_backend. If an HTTP request with this hostname is received, it proxies to buildbot_backend.") 53 alsoRun = flag.String("also_run", "", "Optional path to run as a child process. (Used to run camlistore.org's ./scripts/run-blob-server)") 54 55 pageHtml, errorHtml *template.Template 56 packageHTML *txttemplate.Template 57 ) 58 59 var fmap = template.FuncMap{ 60 "": textFmt, 61 "html": htmlFmt, 62 "htmlesc": htmlEscFmt, 63 } 64 65 // Template formatter for "" (default) format. 66 func textFmt(w io.Writer, format string, x ...interface{}) string { 67 writeAny(w, false, x[0]) 68 return "" 69 } 70 71 // Template formatter for "html" format. 72 func htmlFmt(w io.Writer, format string, x ...interface{}) string { 73 writeAny(w, true, x[0]) 74 return "" 75 } 76 77 // Template formatter for "htmlesc" format. 78 func htmlEscFmt(w io.Writer, format string, x ...interface{}) string { 79 var buf bytes.Buffer 80 writeAny(&buf, false, x[0]) 81 template.HTMLEscape(w, buf.Bytes()) 82 return "" 83 } 84 85 // Write anything to w; optionally html-escaped. 86 func writeAny(w io.Writer, html bool, x interface{}) { 87 switch v := x.(type) { 88 case []byte: 89 writeText(w, v, html) 90 case string: 91 writeText(w, []byte(v), html) 92 default: 93 if html { 94 var buf bytes.Buffer 95 fmt.Fprint(&buf, x) 96 writeText(w, buf.Bytes(), true) 97 } else { 98 fmt.Fprint(w, x) 99 } 100 } 101 } 102 103 // Write text to w; optionally html-escaped. 104 func writeText(w io.Writer, text []byte, html bool) { 105 if html { 106 template.HTMLEscape(w, text) 107 return 108 } 109 w.Write(text) 110 } 111 112 func applyTemplate(t *template.Template, name string, data interface{}) []byte { 113 var buf bytes.Buffer 114 if err := t.Execute(&buf, data); err != nil { 115 log.Printf("%s.Execute: %s", name, err) 116 } 117 return buf.Bytes() 118 } 119 120 func servePage(w http.ResponseWriter, title, subtitle string, content []byte) { 121 // insert an "install command" if it applies 122 if strings.Contains(title, cmdPattern) && subtitle != cmdPattern { 123 toInsert := ` 124 <h3>Installation</h3> 125 <pre>go get camlistore.org/cmd/` + subtitle + `</pre> 126 <h3>Overview</h3><p>` 127 content = bytes.Replace(content, []byte("<p>"), []byte(toInsert), 1) 128 } 129 d := struct { 130 Title string 131 Subtitle string 132 Content template.HTML 133 }{ 134 title, 135 subtitle, 136 template.HTML(content), 137 } 138 139 if err := pageHtml.Execute(w, &d); err != nil { 140 log.Printf("godocHTML.Execute: %s", err) 141 } 142 } 143 144 func readTemplate(name string) *template.Template { 145 fileName := filepath.Join(*root, "tmpl", name) 146 data, err := ioutil.ReadFile(fileName) 147 if err != nil { 148 log.Fatalf("ReadFile %s: %v", fileName, err) 149 } 150 t, err := template.New(name).Funcs(fmap).Parse(string(data)) 151 if err != nil { 152 log.Fatalf("%s: %v", fileName, err) 153 } 154 return t 155 } 156 157 func readTemplates() { 158 pageHtml = readTemplate("page.html") 159 errorHtml = readTemplate("error.html") 160 // TODO(mpl): see about not using text template anymore? 161 packageHTML = readTextTemplate("package.html") 162 } 163 164 func serveError(w http.ResponseWriter, r *http.Request, relpath string, err error) { 165 contents := applyTemplate(errorHtml, "errorHtml", err) // err may contain an absolute path! 166 w.WriteHeader(http.StatusNotFound) 167 servePage(w, "File "+relpath, "", contents) 168 } 169 170 const gerritURLPrefix = "https://camlistore.googlesource.com/camlistore/+/" 171 172 var commitHash = regexp.MustCompile(`^p=camlistore.git;a=commit;h=([0-9a-f]+)$`) 173 174 // empty return value means don't redirect. 175 func redirectPath(u *url.URL) string { 176 // Example: 177 // /code/?p=camlistore.git;a=commit;h=b0d2a8f0e5f27bbfc025a96ec3c7896b42d198ed 178 if strings.HasPrefix(u.Path, "/code/") { 179 m := commitHash.FindStringSubmatch(u.RawQuery) 180 if len(m) == 2 { 181 return gerritURLPrefix + m[1] 182 } 183 } 184 185 if strings.HasPrefix(u.Path, "/gw/") { 186 path := strings.TrimPrefix(u.Path, "/gw/") 187 if strings.HasPrefix(path, "doc") || strings.HasPrefix(path, "clients") { 188 return gerritURLPrefix + "master/" + path 189 } 190 // Assume it's a commit 191 return gerritURLPrefix + path 192 } 193 return "" 194 } 195 196 func mainHandler(rw http.ResponseWriter, req *http.Request) { 197 if target := redirectPath(req.URL); target != "" { 198 http.Redirect(rw, req, target, http.StatusFound) 199 return 200 } 201 202 if dest, ok := issueRedirect(req.URL.Path); ok { 203 http.Redirect(rw, req, dest, http.StatusFound) 204 return 205 } 206 207 relPath := req.URL.Path[1:] // serveFile URL paths start with '/' 208 if strings.Contains(relPath, "..") { 209 return 210 } 211 212 absPath := filepath.Join(*root, "content", relPath) 213 fi, err := os.Lstat(absPath) 214 if err != nil { 215 log.Print(err) 216 serveError(rw, req, relPath, err) 217 return 218 } 219 if fi.IsDir() { 220 relPath += "/index.html" 221 absPath = filepath.Join(*root, "content", relPath) 222 fi, err = os.Lstat(absPath) 223 if err != nil { 224 log.Print(err) 225 serveError(rw, req, relPath, err) 226 return 227 } 228 } 229 230 if !fi.IsDir() { 231 if checkLastModified(rw, req, fi.ModTime()) { 232 return 233 } 234 serveFile(rw, req, relPath, absPath) 235 } 236 } 237 238 // modtime is the modification time of the resource to be served, or IsZero(). 239 // return value is whether this request is now complete. 240 func checkLastModified(w http.ResponseWriter, r *http.Request, modtime time.Time) bool { 241 if modtime.IsZero() { 242 return false 243 } 244 245 // The Date-Modified header truncates sub-second precision, so 246 // use mtime < t+1s instead of mtime <= t to check for unmodified. 247 if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) { 248 h := w.Header() 249 delete(h, "Content-Type") 250 delete(h, "Content-Length") 251 w.WriteHeader(http.StatusNotModified) 252 return true 253 } 254 w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) 255 return false 256 } 257 258 func serveFile(rw http.ResponseWriter, req *http.Request, relPath, absPath string) { 259 data, err := ioutil.ReadFile(absPath) 260 if err != nil { 261 serveError(rw, req, absPath, err) 262 return 263 } 264 265 title := "" 266 if m := h1TitlePattern.FindSubmatch(data); len(m) > 1 { 267 title = string(m[1]) 268 } 269 270 servePage(rw, title, "", data) 271 } 272 273 func isBot(r *http.Request) bool { 274 agent := r.Header.Get("User-Agent") 275 return strings.Contains(agent, "Baidu") || strings.Contains(agent, "bingbot") || 276 strings.Contains(agent, "Ezooms") || strings.Contains(agent, "Googlebot") 277 } 278 279 type noWwwHandler struct { 280 Handler http.Handler 281 } 282 283 func (h *noWwwHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { 284 // Some bots (especially Baidu) don't seem to respect robots.txt and swamp gitweb.cgi, 285 // so explicitly protect it from bots. 286 if ru := r.URL.RequestURI(); strings.Contains(ru, "/code/") && strings.Contains(ru, "?") && isBot(r) { 287 http.Error(rw, "bye", http.StatusUnauthorized) 288 log.Printf("bot denied") 289 return 290 } 291 292 host := strings.ToLower(r.Host) 293 if host == "www.camlistore.org" { 294 http.Redirect(rw, r, "http://camlistore.org"+r.URL.RequestURI(), http.StatusFound) 295 return 296 } 297 h.Handler.ServeHTTP(rw, r) 298 } 299 300 // runAsChild runs res as a child process and 301 // does not wait for it to finish. 302 func runAsChild(res string) { 303 cmdName, err := exec.LookPath(res) 304 if err != nil { 305 log.Fatalf("Could not find %v in $PATH: %v", res, err) 306 } 307 cmd := exec.Command(cmdName) 308 cmd.Stderr = os.Stderr 309 cmd.Stdout = os.Stdout 310 log.Printf("Running %v", res) 311 if err := cmd.Start(); err != nil { 312 log.Fatalf("Program %v failed to start: %v", res, err) 313 } 314 go func() { 315 if err := cmd.Wait(); err != nil { 316 log.Fatalf("Program %s did not end successfully: %v", res, err) 317 } 318 }() 319 } 320 321 func main() { 322 flag.Parse() 323 324 if *root == "" { 325 var err error 326 *root, err = os.Getwd() 327 if err != nil { 328 log.Fatalf("Failed to getwd: %v", err) 329 } 330 } 331 readTemplates() 332 333 mux := http.DefaultServeMux 334 mux.Handle("/favicon.ico", http.FileServer(http.Dir(filepath.Join(*root, "static")))) 335 mux.Handle("/robots.txt", http.FileServer(http.Dir(filepath.Join(*root, "static")))) 336 mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(filepath.Join(*root, "static"))))) 337 mux.Handle("/talks/", http.StripPrefix("/talks/", http.FileServer(http.Dir(filepath.Join(*root, "talks"))))) 338 mux.Handle(pkgPattern, godocHandler{}) 339 mux.Handle(cmdPattern, godocHandler{}) 340 341 mux.HandleFunc("/r/", gerritRedirect) 342 mux.HandleFunc("/debugz/ip", ipHandler) 343 mux.Handle("/docs/contributing", redirTo("/code#contributing")) 344 mux.Handle("/lists", redirTo("/community")) 345 346 mux.HandleFunc("/", mainHandler) 347 348 if *buildbotHost != "" && *buildbotBackend != "" { 349 buildbotUrl, err := url.Parse(*buildbotBackend) 350 if err != nil { 351 log.Fatalf("Failed to parse %v as a URL: %v", *buildbotBackend, err) 352 } 353 buildbotHandler := httputil.NewSingleHostReverseProxy(buildbotUrl) 354 bbhpattern := strings.TrimRight(*buildbotHost, "/") + "/" 355 mux.Handle(bbhpattern, buildbotHandler) 356 } 357 358 var handler http.Handler = &noWwwHandler{Handler: mux} 359 if *logDir != "" || *logStdout { 360 handler = NewLoggingHandler(handler, *logDir, *logStdout) 361 } 362 363 errc := make(chan error) 364 startEmailCommitLoop(errc) 365 366 if *alsoRun != "" { 367 runAsChild(*alsoRun) 368 } 369 370 httpServer := &http.Server{ 371 Addr: *httpAddr, 372 Handler: handler, 373 ReadTimeout: 5 * time.Minute, 374 WriteTimeout: 30 * time.Minute, 375 } 376 go func() { 377 errc <- httpServer.ListenAndServe() 378 }() 379 380 if *httpsAddr != "" { 381 log.Printf("Starting TLS server on %s", *httpsAddr) 382 httpsServer := new(http.Server) 383 *httpsServer = *httpServer 384 httpsServer.Addr = *httpsAddr 385 go func() { 386 errc <- httpsServer.ListenAndServeTLS(*tlsCertFile, *tlsKeyFile) 387 }() 388 } 389 390 log.Fatalf("Serve error: %v", <-errc) 391 } 392 393 var issueNum = regexp.MustCompile(`^/(?:issue(?:s)?|bugs)(/\d*)?$`) 394 395 // issueRedirect returns whether the request should be redirected to the 396 // issues tracker, and the url for that redirection if yes, the empty 397 // string otherwise. 398 func issueRedirect(urlPath string) (string, bool) { 399 m := issueNum.FindStringSubmatch(urlPath) 400 if m == nil { 401 return "", false 402 } 403 issueNumber := strings.TrimPrefix(m[1], "/") 404 suffix := "list" 405 if issueNumber != "" { 406 suffix = "detail?id=" + issueNumber 407 } 408 return "https://code.google.com/p/camlistore/issues/" + suffix, true 409 } 410 411 func gerritRedirect(w http.ResponseWriter, r *http.Request) { 412 dest := "https://camlistore-review.googlesource.com/" 413 if len(r.URL.Path) > len("/r/") { 414 dest += r.URL.Path[1:] 415 } 416 http.Redirect(w, r, dest, http.StatusFound) 417 } 418 419 func redirTo(dest string) http.Handler { 420 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 421 http.Redirect(w, r, dest, http.StatusFound) 422 }) 423 } 424 425 // Not sure what's making these broken URLs like: 426 // 427 // http://localhost:8080/code/?p=camlistore.git%3Bf=doc/json-signing/json-signing.txt%3Bhb=master 428 // 429 // ... but something is. Maybe Buzz? For now just re-write them 430 // . Doesn't seem to be a bug in the CGI implementation, though, which 431 // is what I'd originally suspected. 432 /* 433 func (fu *fixUpGitwebUrls) ServeHTTP(rw http.ResponseWriter, req *http.Request) { 434 oldUrl := req.URL.String() 435 newUrl := strings.Replace(oldUrl, "%3B", ";", -1) 436 if newUrl == oldUrl { 437 fu.handler.ServeHTTP(rw, req) 438 return 439 } 440 http.Redirect(rw, req, newUrl, http.StatusFound) 441 } 442 */ 443 444 func ipHandler(w http.ResponseWriter, r *http.Request) { 445 out, _ := exec.Command("ip", "-f", "inet", "addr", "show", "dev", "eth0").Output() 446 str := string(out) 447 pos := strings.Index(str, "inet ") 448 if pos == -1 { 449 return 450 } 451 str = str[pos+5:] 452 pos = strings.Index(str, "/") 453 if pos == -1 { 454 return 455 } 456 str = str[:pos] 457 w.Write([]byte(str)) 458 }