github.com/olivere/camlistore@v0.0.0-20140121221811-1b7ac2da0199/website/camweb.go (about) 1 /* 2 Copyright 2011 Google Inc. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package main 18 19 import ( 20 "bytes" 21 "flag" 22 "fmt" 23 "html/template" 24 "io" 25 "io/ioutil" 26 "log" 27 "net/http" 28 "net/http/httputil" 29 "net/url" 30 "os" 31 "os/exec" 32 "path/filepath" 33 "regexp" 34 "strings" 35 txttemplate "text/template" 36 "time" 37 ) 38 39 const defaultAddr = ":31798" // default webserver address 40 41 var h1TitlePattern = regexp.MustCompile(`<h1>([^<]+)</h1>`) 42 43 var ( 44 httpAddr = flag.String("http", defaultAddr, "HTTP service address (e.g., '"+defaultAddr+"')") 45 httpsAddr = flag.String("https", "", "HTTPS service address") 46 root = flag.String("root", "", "Website root (parent of 'static', 'content', and 'tmpl") 47 logDir = flag.String("logdir", "", "Directory to write log files to (one per hour), or empty to not log.") 48 logStdout = flag.Bool("logstdout", true, "Write to stdout?") 49 tlsCertFile = flag.String("tlscert", "", "TLS cert file") 50 tlsKeyFile = flag.String("tlskey", "", "TLS private key file") 51 buildbotBackend = flag.String("buildbot_backend", "", "Build bot status backend URL") 52 buildbotHost = flag.String("buildbot_host", "", "Hostname to map to the buildbot_backend. If an HTTP request with this hostname is received, it proxies to buildbot_backend.") 53 alsoRun = flag.String("also_run", "", "Optional path to run as a child process. (Used to run camlistore.org's ./scripts/run-blob-server)") 54 55 pageHtml, errorHtml *template.Template 56 packageHTML *txttemplate.Template 57 ) 58 59 var fmap = template.FuncMap{ 60 "": textFmt, 61 "html": htmlFmt, 62 "htmlesc": htmlEscFmt, 63 } 64 65 // Template formatter for "" (default) format. 66 func textFmt(w io.Writer, format string, x ...interface{}) string { 67 writeAny(w, false, x[0]) 68 return "" 69 } 70 71 // Template formatter for "html" format. 72 func htmlFmt(w io.Writer, format string, x ...interface{}) string { 73 writeAny(w, true, x[0]) 74 return "" 75 } 76 77 // Template formatter for "htmlesc" format. 78 func htmlEscFmt(w io.Writer, format string, x ...interface{}) string { 79 var buf bytes.Buffer 80 writeAny(&buf, false, x[0]) 81 template.HTMLEscape(w, buf.Bytes()) 82 return "" 83 } 84 85 // Write anything to w; optionally html-escaped. 86 func writeAny(w io.Writer, html bool, x interface{}) { 87 switch v := x.(type) { 88 case []byte: 89 writeText(w, v, html) 90 case string: 91 writeText(w, []byte(v), html) 92 default: 93 if html { 94 var buf bytes.Buffer 95 fmt.Fprint(&buf, x) 96 writeText(w, buf.Bytes(), true) 97 } else { 98 fmt.Fprint(w, x) 99 } 100 } 101 } 102 103 // Write text to w; optionally html-escaped. 104 func writeText(w io.Writer, text []byte, html bool) { 105 if html { 106 template.HTMLEscape(w, text) 107 return 108 } 109 w.Write(text) 110 } 111 112 func applyTemplate(t *template.Template, name string, data interface{}) []byte { 113 var buf bytes.Buffer 114 if err := t.Execute(&buf, data); err != nil { 115 log.Printf("%s.Execute: %s", name, err) 116 } 117 return buf.Bytes() 118 } 119 120 func servePage(w http.ResponseWriter, title, subtitle string, content []byte) { 121 // insert an "install command" if it applies 122 if strings.Contains(title, cmdPattern) && subtitle != cmdPattern { 123 toInsert := ` 124 <h3>Installation</h3> 125 <pre>go get camlistore.org/cmd/` + subtitle + `</pre> 126 <h3>Overview</h3><p>` 127 content = bytes.Replace(content, []byte("<p>"), []byte(toInsert), 1) 128 } 129 d := struct { 130 Title string 131 Subtitle string 132 Content template.HTML 133 }{ 134 title, 135 subtitle, 136 template.HTML(content), 137 } 138 139 if err := pageHtml.Execute(w, &d); err != nil { 140 log.Printf("godocHTML.Execute: %s", err) 141 } 142 } 143 144 func readTemplate(name string) *template.Template { 145 fileName := filepath.Join(*root, "tmpl", name) 146 data, err := ioutil.ReadFile(fileName) 147 if err != nil { 148 log.Fatalf("ReadFile %s: %v", fileName, err) 149 } 150 t, err := template.New(name).Funcs(fmap).Parse(string(data)) 151 if err != nil { 152 log.Fatalf("%s: %v", fileName, err) 153 } 154 return t 155 } 156 157 func readTemplates() { 158 pageHtml = readTemplate("page.html") 159 errorHtml = readTemplate("error.html") 160 // TODO(mpl): see about not using text template anymore? 161 packageHTML = readTextTemplate("package.html") 162 } 163 164 func serveError(w http.ResponseWriter, r *http.Request, relpath string, err error) { 165 contents := applyTemplate(errorHtml, "errorHtml", err) // err may contain an absolute path! 166 w.WriteHeader(http.StatusNotFound) 167 servePage(w, "File "+relpath, "", contents) 168 } 169 170 const gerritURLPrefix = "https://camlistore.googlesource.com/camlistore/+/" 171 172 var commitHash = regexp.MustCompile(`^p=camlistore.git;a=commit;h=([0-9a-f]+)$`) 173 174 // empty return value means don't redirect. 175 func redirectPath(u *url.URL) string { 176 // Example: 177 // /code/?p=camlistore.git;a=commit;h=b0d2a8f0e5f27bbfc025a96ec3c7896b42d198ed 178 if strings.HasPrefix(u.Path, "/code/") { 179 m := commitHash.FindStringSubmatch(u.RawQuery) 180 if len(m) == 2 { 181 return gerritURLPrefix + m[1] 182 } 183 } 184 185 if strings.HasPrefix(u.Path, "/gw/") { 186 path := strings.TrimPrefix(u.Path, "/gw/") 187 if strings.HasPrefix(path, "doc") || strings.HasPrefix(path, "clients") { 188 return gerritURLPrefix + "master/" + path 189 } 190 // Assume it's a commit 191 return gerritURLPrefix + path 192 } 193 return "" 194 } 195 196 func mainHandler(rw http.ResponseWriter, req *http.Request) { 197 if target := redirectPath(req.URL); target != "" { 198 http.Redirect(rw, req, target, http.StatusFound) 199 return 200 } 201 202 if dest, ok := issueRedirect(req.URL.Path); ok { 203 http.Redirect(rw, req, dest, http.StatusFound) 204 return 205 } 206 207 relPath := req.URL.Path[1:] // serveFile URL paths start with '/' 208 if strings.Contains(relPath, "..") { 209 return 210 } 211 212 absPath := filepath.Join(*root, "content", relPath) 213 fi, err := os.Lstat(absPath) 214 if err != nil { 215 log.Print(err) 216 serveError(rw, req, relPath, err) 217 return 218 } 219 if fi.IsDir() { 220 relPath += "/index.html" 221 absPath = filepath.Join(*root, "content", relPath) 222 fi, err = os.Lstat(absPath) 223 if err != nil { 224 log.Print(err) 225 serveError(rw, req, relPath, err) 226 return 227 } 228 } 229 230 if !fi.IsDir() { 231 if checkLastModified(rw, req, fi.ModTime()) { 232 return 233 } 234 serveFile(rw, req, relPath, absPath) 235 } 236 } 237 238 // modtime is the modification time of the resource to be served, or IsZero(). 239 // return value is whether this request is now complete. 240 func checkLastModified(w http.ResponseWriter, r *http.Request, modtime time.Time) bool { 241 if modtime.IsZero() { 242 return false 243 } 244 245 // The Date-Modified header truncates sub-second precision, so 246 // use mtime < t+1s instead of mtime <= t to check for unmodified. 247 if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) { 248 h := w.Header() 249 delete(h, "Content-Type") 250 delete(h, "Content-Length") 251 w.WriteHeader(http.StatusNotModified) 252 return true 253 } 254 w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) 255 return false 256 } 257 258 func serveFile(rw http.ResponseWriter, req *http.Request, relPath, absPath string) { 259 data, err := ioutil.ReadFile(absPath) 260 if err != nil { 261 serveError(rw, req, absPath, err) 262 return 263 } 264 265 title := "" 266 if m := h1TitlePattern.FindSubmatch(data); len(m) > 1 { 267 title = string(m[1]) 268 } 269 270 servePage(rw, title, "", data) 271 } 272 273 func isBot(r *http.Request) bool { 274 agent := r.Header.Get("User-Agent") 275 return strings.Contains(agent, "Baidu") || strings.Contains(agent, "bingbot") || 276 strings.Contains(agent, "Ezooms") || strings.Contains(agent, "Googlebot") 277 } 278 279 type noWwwHandler struct { 280 Handler http.Handler 281 } 282 283 func (h *noWwwHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) { 284 // Some bots (especially Baidu) don't seem to respect robots.txt and swamp gitweb.cgi, 285 // so explicitly protect it from bots. 286 if ru := r.URL.RequestURI(); strings.Contains(ru, "/code/") && strings.Contains(ru, "?") && isBot(r) { 287 http.Error(rw, "bye", http.StatusUnauthorized) 288 log.Printf("bot denied") 289 return 290 } 291 292 host := strings.ToLower(r.Host) 293 if host == "www.camlistore.org" { 294 http.Redirect(rw, r, "http://camlistore.org"+r.URL.RequestURI(), http.StatusFound) 295 return 296 } 297 h.Handler.ServeHTTP(rw, r) 298 } 299 300 // runAsChild runs res as a child process and 301 // does not wait for it to finish. 302 func runAsChild(res string) { 303 cmdName, err := exec.LookPath(res) 304 if err != nil { 305 log.Fatalf("Could not find %v in $PATH: %v", res, err) 306 } 307 cmd := exec.Command(cmdName) 308 cmd.Stderr = os.Stderr 309 cmd.Stdout = os.Stdout 310 log.Printf("Running %v", res) 311 if err := cmd.Start(); err != nil { 312 log.Fatalf("Program %v failed to start: %v", res, err) 313 } 314 go func() { 315 if err := cmd.Wait(); err != nil { 316 log.Fatalf("Program %s did not end successfully: %v", res, err) 317 } 318 }() 319 } 320 321 func main() { 322 flag.Parse() 323 324 if *root == "" { 325 var err error 326 *root, err = os.Getwd() 327 if err != nil { 328 log.Fatalf("Failed to getwd: %v", err) 329 } 330 } 331 readTemplates() 332 333 mux := http.DefaultServeMux 334 mux.Handle("/favicon.ico", http.FileServer(http.Dir(filepath.Join(*root, "static")))) 335 mux.Handle("/robots.txt", http.FileServer(http.Dir(filepath.Join(*root, "static")))) 336 mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(filepath.Join(*root, "static"))))) 337 mux.Handle("/talks/", http.StripPrefix("/talks/", http.FileServer(http.Dir(filepath.Join(*root, "talks"))))) 338 mux.Handle(pkgPattern, godocHandler{}) 339 mux.Handle(cmdPattern, godocHandler{}) 340 341 mux.HandleFunc("/r/", gerritRedirect) 342 mux.HandleFunc("/debugz/ip", ipHandler) 343 344 mux.HandleFunc("/", mainHandler) 345 346 if *buildbotHost != "" && *buildbotBackend != "" { 347 buildbotUrl, err := url.Parse(*buildbotBackend) 348 if err != nil { 349 log.Fatalf("Failed to parse %v as a URL: %v", *buildbotBackend, err) 350 } 351 buildbotHandler := httputil.NewSingleHostReverseProxy(buildbotUrl) 352 bbhpattern := strings.TrimRight(*buildbotHost, "/") + "/" 353 mux.Handle(bbhpattern, buildbotHandler) 354 } 355 356 var handler http.Handler = &noWwwHandler{Handler: mux} 357 if *logDir != "" || *logStdout { 358 handler = NewLoggingHandler(handler, *logDir, *logStdout) 359 } 360 361 errc := make(chan error) 362 startEmailCommitLoop(errc) 363 364 if *alsoRun != "" { 365 runAsChild(*alsoRun) 366 } 367 368 httpServer := &http.Server{ 369 Addr: *httpAddr, 370 Handler: handler, 371 ReadTimeout: 5 * time.Minute, 372 WriteTimeout: 30 * time.Minute, 373 } 374 go func() { 375 errc <- httpServer.ListenAndServe() 376 }() 377 378 if *httpsAddr != "" { 379 log.Printf("Starting TLS server on %s", *httpsAddr) 380 httpsServer := new(http.Server) 381 *httpsServer = *httpServer 382 httpsServer.Addr = *httpsAddr 383 go func() { 384 errc <- httpsServer.ListenAndServeTLS(*tlsCertFile, *tlsKeyFile) 385 }() 386 } 387 388 log.Fatalf("Serve error: %v", <-errc) 389 } 390 391 var issueNum = regexp.MustCompile(`^/(?:issue(?:s)?|bugs)(/\d*)?$`) 392 393 // issueRedirect returns whether the request should be redirected to the 394 // issues tracker, and the url for that redirection if yes, the empty 395 // string otherwise. 396 func issueRedirect(urlPath string) (string, bool) { 397 m := issueNum.FindStringSubmatch(urlPath) 398 if m == nil { 399 return "", false 400 } 401 issueNumber := strings.TrimPrefix(m[1], "/") 402 suffix := "list" 403 if issueNumber != "" { 404 suffix = "detail?id=" + issueNumber 405 } 406 return "https://code.google.com/p/camlistore/issues/" + suffix, true 407 } 408 409 func gerritRedirect(w http.ResponseWriter, r *http.Request) { 410 dest := "https://camlistore-review.googlesource.com/" 411 if len(r.URL.Path) > len("/r/") { 412 dest += r.URL.Path[1:] 413 } 414 http.Redirect(w, r, dest, http.StatusFound) 415 } 416 417 // Not sure what's making these broken URLs like: 418 // 419 // http://localhost:8080/code/?p=camlistore.git%3Bf=doc/json-signing/json-signing.txt%3Bhb=master 420 // 421 // ... but something is. Maybe Buzz? For now just re-write them 422 // . Doesn't seem to be a bug in the CGI implementation, though, which 423 // is what I'd originally suspected. 424 /* 425 func (fu *fixUpGitwebUrls) ServeHTTP(rw http.ResponseWriter, req *http.Request) { 426 oldUrl := req.URL.String() 427 newUrl := strings.Replace(oldUrl, "%3B", ";", -1) 428 if newUrl == oldUrl { 429 fu.handler.ServeHTTP(rw, req) 430 return 431 } 432 http.Redirect(rw, req, newUrl, http.StatusFound) 433 } 434 */ 435 436 func ipHandler(w http.ResponseWriter, r *http.Request) { 437 out, _ := exec.Command("ip", "-f", "inet", "addr", "show", "dev", "eth0").Output() 438 str := string(out) 439 pos := strings.Index(str, "inet ") 440 if pos == -1 { 441 return 442 } 443 str = str[pos+5:] 444 pos = strings.Index(str, "/") 445 if pos == -1 { 446 return 447 } 448 str = str[:pos] 449 w.Write([]byte(str)) 450 }