github.com/olivere/camlistore@v0.0.0-20140121221811-1b7ac2da0199/website/camweb.go (about)

     1  /*
     2  Copyright 2011 Google Inc.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8       http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package main
    18  
    19  import (
    20  	"bytes"
    21  	"flag"
    22  	"fmt"
    23  	"html/template"
    24  	"io"
    25  	"io/ioutil"
    26  	"log"
    27  	"net/http"
    28  	"net/http/httputil"
    29  	"net/url"
    30  	"os"
    31  	"os/exec"
    32  	"path/filepath"
    33  	"regexp"
    34  	"strings"
    35  	txttemplate "text/template"
    36  	"time"
    37  )
    38  
    39  const defaultAddr = ":31798" // default webserver address
    40  
    41  var h1TitlePattern = regexp.MustCompile(`<h1>([^<]+)</h1>`)
    42  
    43  var (
    44  	httpAddr        = flag.String("http", defaultAddr, "HTTP service address (e.g., '"+defaultAddr+"')")
    45  	httpsAddr       = flag.String("https", "", "HTTPS service address")
    46  	root            = flag.String("root", "", "Website root (parent of 'static', 'content', and 'tmpl")
    47  	logDir          = flag.String("logdir", "", "Directory to write log files to (one per hour), or empty to not log.")
    48  	logStdout       = flag.Bool("logstdout", true, "Write to stdout?")
    49  	tlsCertFile     = flag.String("tlscert", "", "TLS cert file")
    50  	tlsKeyFile      = flag.String("tlskey", "", "TLS private key file")
    51  	buildbotBackend = flag.String("buildbot_backend", "", "Build bot status backend URL")
    52  	buildbotHost    = flag.String("buildbot_host", "", "Hostname to map to the buildbot_backend. If an HTTP request with this hostname is received, it proxies to buildbot_backend.")
    53  	alsoRun         = flag.String("also_run", "", "Optional path to run as a child process. (Used to run camlistore.org's ./scripts/run-blob-server)")
    54  
    55  	pageHtml, errorHtml *template.Template
    56  	packageHTML         *txttemplate.Template
    57  )
    58  
    59  var fmap = template.FuncMap{
    60  	"":        textFmt,
    61  	"html":    htmlFmt,
    62  	"htmlesc": htmlEscFmt,
    63  }
    64  
    65  // Template formatter for "" (default) format.
    66  func textFmt(w io.Writer, format string, x ...interface{}) string {
    67  	writeAny(w, false, x[0])
    68  	return ""
    69  }
    70  
    71  // Template formatter for "html" format.
    72  func htmlFmt(w io.Writer, format string, x ...interface{}) string {
    73  	writeAny(w, true, x[0])
    74  	return ""
    75  }
    76  
    77  // Template formatter for "htmlesc" format.
    78  func htmlEscFmt(w io.Writer, format string, x ...interface{}) string {
    79  	var buf bytes.Buffer
    80  	writeAny(&buf, false, x[0])
    81  	template.HTMLEscape(w, buf.Bytes())
    82  	return ""
    83  }
    84  
    85  // Write anything to w; optionally html-escaped.
    86  func writeAny(w io.Writer, html bool, x interface{}) {
    87  	switch v := x.(type) {
    88  	case []byte:
    89  		writeText(w, v, html)
    90  	case string:
    91  		writeText(w, []byte(v), html)
    92  	default:
    93  		if html {
    94  			var buf bytes.Buffer
    95  			fmt.Fprint(&buf, x)
    96  			writeText(w, buf.Bytes(), true)
    97  		} else {
    98  			fmt.Fprint(w, x)
    99  		}
   100  	}
   101  }
   102  
   103  // Write text to w; optionally html-escaped.
   104  func writeText(w io.Writer, text []byte, html bool) {
   105  	if html {
   106  		template.HTMLEscape(w, text)
   107  		return
   108  	}
   109  	w.Write(text)
   110  }
   111  
   112  func applyTemplate(t *template.Template, name string, data interface{}) []byte {
   113  	var buf bytes.Buffer
   114  	if err := t.Execute(&buf, data); err != nil {
   115  		log.Printf("%s.Execute: %s", name, err)
   116  	}
   117  	return buf.Bytes()
   118  }
   119  
   120  func servePage(w http.ResponseWriter, title, subtitle string, content []byte) {
   121  	// insert an "install command" if it applies
   122  	if strings.Contains(title, cmdPattern) && subtitle != cmdPattern {
   123  		toInsert := `
   124  		<h3>Installation</h3>
   125  		<pre>go get camlistore.org/cmd/` + subtitle + `</pre>
   126  		<h3>Overview</h3><p>`
   127  		content = bytes.Replace(content, []byte("<p>"), []byte(toInsert), 1)
   128  	}
   129  	d := struct {
   130  		Title    string
   131  		Subtitle string
   132  		Content  template.HTML
   133  	}{
   134  		title,
   135  		subtitle,
   136  		template.HTML(content),
   137  	}
   138  
   139  	if err := pageHtml.Execute(w, &d); err != nil {
   140  		log.Printf("godocHTML.Execute: %s", err)
   141  	}
   142  }
   143  
   144  func readTemplate(name string) *template.Template {
   145  	fileName := filepath.Join(*root, "tmpl", name)
   146  	data, err := ioutil.ReadFile(fileName)
   147  	if err != nil {
   148  		log.Fatalf("ReadFile %s: %v", fileName, err)
   149  	}
   150  	t, err := template.New(name).Funcs(fmap).Parse(string(data))
   151  	if err != nil {
   152  		log.Fatalf("%s: %v", fileName, err)
   153  	}
   154  	return t
   155  }
   156  
   157  func readTemplates() {
   158  	pageHtml = readTemplate("page.html")
   159  	errorHtml = readTemplate("error.html")
   160  	// TODO(mpl): see about not using text template anymore?
   161  	packageHTML = readTextTemplate("package.html")
   162  }
   163  
   164  func serveError(w http.ResponseWriter, r *http.Request, relpath string, err error) {
   165  	contents := applyTemplate(errorHtml, "errorHtml", err) // err may contain an absolute path!
   166  	w.WriteHeader(http.StatusNotFound)
   167  	servePage(w, "File "+relpath, "", contents)
   168  }
   169  
   170  const gerritURLPrefix = "https://camlistore.googlesource.com/camlistore/+/"
   171  
   172  var commitHash = regexp.MustCompile(`^p=camlistore.git;a=commit;h=([0-9a-f]+)$`)
   173  
   174  // empty return value means don't redirect.
   175  func redirectPath(u *url.URL) string {
   176  	// Example:
   177  	// /code/?p=camlistore.git;a=commit;h=b0d2a8f0e5f27bbfc025a96ec3c7896b42d198ed
   178  	if strings.HasPrefix(u.Path, "/code/") {
   179  		m := commitHash.FindStringSubmatch(u.RawQuery)
   180  		if len(m) == 2 {
   181  			return gerritURLPrefix + m[1]
   182  		}
   183  	}
   184  
   185  	if strings.HasPrefix(u.Path, "/gw/") {
   186  		path := strings.TrimPrefix(u.Path, "/gw/")
   187  		if strings.HasPrefix(path, "doc") || strings.HasPrefix(path, "clients") {
   188  			return gerritURLPrefix + "master/" + path
   189  		}
   190  		// Assume it's a commit
   191  		return gerritURLPrefix + path
   192  	}
   193  	return ""
   194  }
   195  
   196  func mainHandler(rw http.ResponseWriter, req *http.Request) {
   197  	if target := redirectPath(req.URL); target != "" {
   198  		http.Redirect(rw, req, target, http.StatusFound)
   199  		return
   200  	}
   201  
   202  	if dest, ok := issueRedirect(req.URL.Path); ok {
   203  		http.Redirect(rw, req, dest, http.StatusFound)
   204  		return
   205  	}
   206  
   207  	relPath := req.URL.Path[1:] // serveFile URL paths start with '/'
   208  	if strings.Contains(relPath, "..") {
   209  		return
   210  	}
   211  
   212  	absPath := filepath.Join(*root, "content", relPath)
   213  	fi, err := os.Lstat(absPath)
   214  	if err != nil {
   215  		log.Print(err)
   216  		serveError(rw, req, relPath, err)
   217  		return
   218  	}
   219  	if fi.IsDir() {
   220  		relPath += "/index.html"
   221  		absPath = filepath.Join(*root, "content", relPath)
   222  		fi, err = os.Lstat(absPath)
   223  		if err != nil {
   224  			log.Print(err)
   225  			serveError(rw, req, relPath, err)
   226  			return
   227  		}
   228  	}
   229  
   230  	if !fi.IsDir() {
   231  		if checkLastModified(rw, req, fi.ModTime()) {
   232  			return
   233  		}
   234  		serveFile(rw, req, relPath, absPath)
   235  	}
   236  }
   237  
   238  // modtime is the modification time of the resource to be served, or IsZero().
   239  // return value is whether this request is now complete.
   240  func checkLastModified(w http.ResponseWriter, r *http.Request, modtime time.Time) bool {
   241  	if modtime.IsZero() {
   242  		return false
   243  	}
   244  
   245  	// The Date-Modified header truncates sub-second precision, so
   246  	// use mtime < t+1s instead of mtime <= t to check for unmodified.
   247  	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) {
   248  		h := w.Header()
   249  		delete(h, "Content-Type")
   250  		delete(h, "Content-Length")
   251  		w.WriteHeader(http.StatusNotModified)
   252  		return true
   253  	}
   254  	w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
   255  	return false
   256  }
   257  
   258  func serveFile(rw http.ResponseWriter, req *http.Request, relPath, absPath string) {
   259  	data, err := ioutil.ReadFile(absPath)
   260  	if err != nil {
   261  		serveError(rw, req, absPath, err)
   262  		return
   263  	}
   264  
   265  	title := ""
   266  	if m := h1TitlePattern.FindSubmatch(data); len(m) > 1 {
   267  		title = string(m[1])
   268  	}
   269  
   270  	servePage(rw, title, "", data)
   271  }
   272  
   273  func isBot(r *http.Request) bool {
   274  	agent := r.Header.Get("User-Agent")
   275  	return strings.Contains(agent, "Baidu") || strings.Contains(agent, "bingbot") ||
   276  		strings.Contains(agent, "Ezooms") || strings.Contains(agent, "Googlebot")
   277  }
   278  
   279  type noWwwHandler struct {
   280  	Handler http.Handler
   281  }
   282  
   283  func (h *noWwwHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
   284  	// Some bots (especially Baidu) don't seem to respect robots.txt and swamp gitweb.cgi,
   285  	// so explicitly protect it from bots.
   286  	if ru := r.URL.RequestURI(); strings.Contains(ru, "/code/") && strings.Contains(ru, "?") && isBot(r) {
   287  		http.Error(rw, "bye", http.StatusUnauthorized)
   288  		log.Printf("bot denied")
   289  		return
   290  	}
   291  
   292  	host := strings.ToLower(r.Host)
   293  	if host == "www.camlistore.org" {
   294  		http.Redirect(rw, r, "http://camlistore.org"+r.URL.RequestURI(), http.StatusFound)
   295  		return
   296  	}
   297  	h.Handler.ServeHTTP(rw, r)
   298  }
   299  
   300  // runAsChild runs res as a child process and
   301  // does not wait for it to finish.
   302  func runAsChild(res string) {
   303  	cmdName, err := exec.LookPath(res)
   304  	if err != nil {
   305  		log.Fatalf("Could not find %v in $PATH: %v", res, err)
   306  	}
   307  	cmd := exec.Command(cmdName)
   308  	cmd.Stderr = os.Stderr
   309  	cmd.Stdout = os.Stdout
   310  	log.Printf("Running %v", res)
   311  	if err := cmd.Start(); err != nil {
   312  		log.Fatalf("Program %v failed to start: %v", res, err)
   313  	}
   314  	go func() {
   315  		if err := cmd.Wait(); err != nil {
   316  			log.Fatalf("Program %s did not end successfully: %v", res, err)
   317  		}
   318  	}()
   319  }
   320  
   321  func main() {
   322  	flag.Parse()
   323  
   324  	if *root == "" {
   325  		var err error
   326  		*root, err = os.Getwd()
   327  		if err != nil {
   328  			log.Fatalf("Failed to getwd: %v", err)
   329  		}
   330  	}
   331  	readTemplates()
   332  
   333  	mux := http.DefaultServeMux
   334  	mux.Handle("/favicon.ico", http.FileServer(http.Dir(filepath.Join(*root, "static"))))
   335  	mux.Handle("/robots.txt", http.FileServer(http.Dir(filepath.Join(*root, "static"))))
   336  	mux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(filepath.Join(*root, "static")))))
   337  	mux.Handle("/talks/", http.StripPrefix("/talks/", http.FileServer(http.Dir(filepath.Join(*root, "talks")))))
   338  	mux.Handle(pkgPattern, godocHandler{})
   339  	mux.Handle(cmdPattern, godocHandler{})
   340  
   341  	mux.HandleFunc("/r/", gerritRedirect)
   342  	mux.HandleFunc("/debugz/ip", ipHandler)
   343  
   344  	mux.HandleFunc("/", mainHandler)
   345  
   346  	if *buildbotHost != "" && *buildbotBackend != "" {
   347  		buildbotUrl, err := url.Parse(*buildbotBackend)
   348  		if err != nil {
   349  			log.Fatalf("Failed to parse %v as a URL: %v", *buildbotBackend, err)
   350  		}
   351  		buildbotHandler := httputil.NewSingleHostReverseProxy(buildbotUrl)
   352  		bbhpattern := strings.TrimRight(*buildbotHost, "/") + "/"
   353  		mux.Handle(bbhpattern, buildbotHandler)
   354  	}
   355  
   356  	var handler http.Handler = &noWwwHandler{Handler: mux}
   357  	if *logDir != "" || *logStdout {
   358  		handler = NewLoggingHandler(handler, *logDir, *logStdout)
   359  	}
   360  
   361  	errc := make(chan error)
   362  	startEmailCommitLoop(errc)
   363  
   364  	if *alsoRun != "" {
   365  		runAsChild(*alsoRun)
   366  	}
   367  
   368  	httpServer := &http.Server{
   369  		Addr:         *httpAddr,
   370  		Handler:      handler,
   371  		ReadTimeout:  5 * time.Minute,
   372  		WriteTimeout: 30 * time.Minute,
   373  	}
   374  	go func() {
   375  		errc <- httpServer.ListenAndServe()
   376  	}()
   377  
   378  	if *httpsAddr != "" {
   379  		log.Printf("Starting TLS server on %s", *httpsAddr)
   380  		httpsServer := new(http.Server)
   381  		*httpsServer = *httpServer
   382  		httpsServer.Addr = *httpsAddr
   383  		go func() {
   384  			errc <- httpsServer.ListenAndServeTLS(*tlsCertFile, *tlsKeyFile)
   385  		}()
   386  	}
   387  
   388  	log.Fatalf("Serve error: %v", <-errc)
   389  }
   390  
   391  var issueNum = regexp.MustCompile(`^/(?:issue(?:s)?|bugs)(/\d*)?$`)
   392  
   393  // issueRedirect returns whether the request should be redirected to the
   394  // issues tracker, and the url for that redirection if yes, the empty
   395  // string otherwise.
   396  func issueRedirect(urlPath string) (string, bool) {
   397  	m := issueNum.FindStringSubmatch(urlPath)
   398  	if m == nil {
   399  		return "", false
   400  	}
   401  	issueNumber := strings.TrimPrefix(m[1], "/")
   402  	suffix := "list"
   403  	if issueNumber != "" {
   404  		suffix = "detail?id=" + issueNumber
   405  	}
   406  	return "https://code.google.com/p/camlistore/issues/" + suffix, true
   407  }
   408  
   409  func gerritRedirect(w http.ResponseWriter, r *http.Request) {
   410  	dest := "https://camlistore-review.googlesource.com/"
   411  	if len(r.URL.Path) > len("/r/") {
   412  		dest += r.URL.Path[1:]
   413  	}
   414  	http.Redirect(w, r, dest, http.StatusFound)
   415  }
   416  
   417  // Not sure what's making these broken URLs like:
   418  //
   419  //   http://localhost:8080/code/?p=camlistore.git%3Bf=doc/json-signing/json-signing.txt%3Bhb=master
   420  //
   421  // ... but something is.  Maybe Buzz?  For now just re-write them
   422  // . Doesn't seem to be a bug in the CGI implementation, though, which
   423  // is what I'd originally suspected.
   424  /*
   425  func (fu *fixUpGitwebUrls) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
   426  	oldUrl := req.URL.String()
   427  	newUrl := strings.Replace(oldUrl, "%3B", ";", -1)
   428  	if newUrl == oldUrl {
   429  		fu.handler.ServeHTTP(rw, req)
   430  		return
   431  	}
   432  	http.Redirect(rw, req, newUrl, http.StatusFound)
   433  }
   434  */
   435  
   436  func ipHandler(w http.ResponseWriter, r *http.Request) {
   437  	out, _ := exec.Command("ip", "-f", "inet", "addr", "show", "dev", "eth0").Output()
   438  	str := string(out)
   439  	pos := strings.Index(str, "inet ")
   440  	if pos == -1 {
   441  		return
   442  	}
   443  	str = str[pos+5:]
   444  	pos = strings.Index(str, "/")
   445  	if pos == -1 {
   446  		return
   447  	}
   448  	str = str[:pos]
   449  	w.Write([]byte(str))
   450  }