golang.org/x/build@v0.0.0-20240506185731-218518f32b70/cmd/pubsubhelper/pubsubhelper.go (about)

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // The pubsubhelper is an SMTP server for Gerrit updates and an HTTP
     6  // server for Github webhook updates. It then lets other clients subscribe
     7  // to those changes.
     8  package main
     9  
    10  import (
    11  	"bufio"
    12  	"bytes"
    13  	"context"
    14  	"encoding/json"
    15  	"errors"
    16  	"flag"
    17  	"fmt"
    18  	"io"
    19  	"log"
    20  	"net"
    21  	"net/http"
    22  	"net/textproto"
    23  	"os"
    24  	"os/signal"
    25  	"strconv"
    26  	"strings"
    27  	"sync"
    28  	"time"
    29  
    30  	"github.com/bradfitz/go-smtpd/smtpd"
    31  	"github.com/jellevandenhooff/dkim"
    32  	"go4.org/types"
    33  	"golang.org/x/build/cmd/pubsubhelper/pubsubtypes"
    34  	"golang.org/x/build/internal/https"
    35  	"golang.org/x/build/internal/secret"
    36  )
    37  
    38  var (
    39  	botEmail      = flag.String("rcpt", "\x67\x6f\x70\x68\x65\x72\x62\x6f\x74@pubsubhelper.golang.org", "email address of bot. incoming emails must be to this address.")
    40  	smtpListen    = flag.String("smtp", ":25", "SMTP listen address")
    41  	webhookSecret = flag.String("webhook-secret", "", "Development mode GitHub webhook secret. This flag should not be used in production.")
    42  )
    43  
    44  func main() {
    45  	https.RegisterFlags(flag.CommandLine)
    46  	flag.Parse()
    47  
    48  	ctx := context.Background()
    49  	ch := make(chan os.Signal, 1)
    50  	signal.Notify(ch, os.Interrupt)
    51  	go func() {
    52  		sig := <-ch
    53  		log.Printf("Signal %v received; exiting with status 0.", sig)
    54  		os.Exit(0)
    55  	}()
    56  
    57  	// webhooksecret should not be set in production
    58  	if *webhookSecret == "" {
    59  		sc := secret.MustNewClient()
    60  		defer sc.Close()
    61  
    62  		ctxSc, cancel := context.WithTimeout(ctx, 10*time.Second)
    63  		defer cancel()
    64  
    65  		var err error
    66  		*webhookSecret, err = sc.Retrieve(ctxSc, secret.NamePubSubHelperWebhook)
    67  		if err != nil {
    68  			log.Fatalf("unable to retrieve webhook secret %v", err)
    69  		}
    70  	}
    71  
    72  	http.HandleFunc("/", handleRoot)
    73  	http.HandleFunc("/waitevent", handleWaitEvent)
    74  	http.HandleFunc("/recent", handleRecent)
    75  	http.HandleFunc("/github-webhook", handleGithubWebhook)
    76  
    77  	errc := make(chan error)
    78  	go func() {
    79  		log.Printf("running pubsubhelper on %s", *smtpListen)
    80  		s := &smtpd.Server{
    81  			Addr:            *smtpListen,
    82  			OnNewMail:       onNewMail,
    83  			OnNewConnection: onNewConnection,
    84  			ReadTimeout:     time.Minute,
    85  		}
    86  		err := s.ListenAndServe()
    87  		errc <- fmt.Errorf("SMTP ListenAndServe: %v", err)
    88  	}()
    89  	log.Fatalln(https.ListenAndServe(ctx, http.DefaultServeMux))
    90  }
    91  
    92  func handleRoot(w http.ResponseWriter, r *http.Request) {
    93  	if r.URL.Path != "/" {
    94  		http.NotFound(w, r)
    95  		return
    96  	}
    97  	io.WriteString(w, `<html>
    98  <body>
    99    This is <a href="https://godoc.org/golang.org/x/build/cmd/pubsubhelper">pubsubhelper</a>.
   100  
   101  <ul>
   102     <li><b><a href="/waitevent">/waitevent</a></b>: long-poll wait 30s for next event (use ?after=[RFC3339Nano] to resume at point)</li>
   103     <li><b><a href="/recent">/recent</a></b>: recent events, without long-polling.</li>
   104  </ul>
   105  
   106  </body>
   107  </html>
   108  `)
   109  }
   110  
   111  func handleWaitEvent(w http.ResponseWriter, r *http.Request) {
   112  	if r.Method != "GET" {
   113  		http.Error(w, "requires GET", http.StatusBadRequest)
   114  		return
   115  	}
   116  
   117  	ch := make(chan *eventAndJSON, 1)
   118  
   119  	var after time.Time
   120  	if v := r.FormValue("after"); v != "" {
   121  		var err error
   122  		after, err = time.Parse(time.RFC3339Nano, v)
   123  		if err != nil {
   124  			http.Error(w, "'after' parameter is not in time.RFC3339Nano format", http.StatusBadRequest)
   125  			return
   126  		}
   127  	} else {
   128  		after = time.Now()
   129  	}
   130  
   131  	register(ch, after)
   132  	defer unregister(ch)
   133  	ctx := r.Context()
   134  
   135  	timer := time.NewTimer(30 * time.Second)
   136  	defer timer.Stop()
   137  
   138  	var e *eventAndJSON
   139  	select {
   140  	case <-ctx.Done():
   141  		return
   142  	case <-timer.C:
   143  		e = newEventAndJSON(&pubsubtypes.Event{
   144  			LongPollTimeout: true,
   145  		})
   146  	case e = <-ch:
   147  	}
   148  
   149  	w.Header().Set("Content-Type", "application/json; charset=utf-8")
   150  	io.WriteString(w, e.json)
   151  }
   152  
   153  func handleRecent(w http.ResponseWriter, r *http.Request) {
   154  	w.Header().Set("Content-Type", "application/json; charset=utf-8")
   155  
   156  	var after time.Time
   157  	if v := r.FormValue("after"); v != "" {
   158  		var err error
   159  		after, err = time.Parse(time.RFC3339Nano, v)
   160  		if err != nil {
   161  			http.Error(w, "'after' parameter is not in time.RFC3339Nano format", http.StatusBadRequest)
   162  			return
   163  		}
   164  	}
   165  
   166  	var buf bytes.Buffer
   167  	mu.Lock()
   168  	buf.WriteString("[\n")
   169  	n := 0
   170  	for i := len(recent) - 1; i >= 0; i-- {
   171  		ev := recent[i]
   172  		if ev.Time.Time().Before(after) {
   173  			continue
   174  		}
   175  		if n > 0 {
   176  			buf.WriteString(",\n")
   177  		}
   178  		n++
   179  		buf.WriteString(ev.json)
   180  	}
   181  	buf.WriteString("\n]\n")
   182  	mu.Unlock()
   183  	w.Write(buf.Bytes())
   184  }
   185  
   186  type env struct {
   187  	from    smtpd.MailAddress
   188  	body    bytes.Buffer
   189  	conn    smtpd.Connection
   190  	tooBig  bool
   191  	hasRcpt bool
   192  }
   193  
   194  func (e *env) BeginData() error {
   195  	if !e.hasRcpt {
   196  		return smtpd.SMTPError("554 5.5.1 Error: no valid recipients")
   197  	}
   198  	return nil
   199  }
   200  
   201  func (e *env) AddRecipient(rcpt smtpd.MailAddress) error {
   202  	if e.hasRcpt {
   203  		return smtpd.SMTPError("554 5.5.1 Error: dup recipients")
   204  	}
   205  	to := rcpt.Email()
   206  	if to != *botEmail {
   207  		return errors.New("bogus recipient")
   208  	}
   209  	e.hasRcpt = true
   210  	return nil
   211  }
   212  
   213  func (e *env) Write(line []byte) error {
   214  	const maxSize = 5 << 20
   215  	if e.body.Len() > maxSize {
   216  		e.tooBig = true
   217  		return nil
   218  	}
   219  	e.body.Write(line)
   220  	return nil
   221  }
   222  
   223  var (
   224  	headerSep           = []byte("\r\n\r\n")
   225  	dkimSignatureHeader = []byte("\nDKIM-Signature:")
   226  )
   227  
   228  func (e *env) Close() error {
   229  	if e.tooBig {
   230  		log.Printf("Ignoring too-large email from %q", e.from)
   231  		return nil
   232  	}
   233  	from := e.from.Email()
   234  	bodyBytes := e.body.Bytes()
   235  	if !bytes.Contains(bodyBytes, dkimSignatureHeader) {
   236  		log.Printf("Ignoring unsigned (~spam) email from %q", from)
   237  		return nil
   238  	}
   239  
   240  	headerBytes := bodyBytes
   241  	if i := bytes.Index(headerBytes, headerSep); i == -1 {
   242  		log.Printf("Ignoring email without header separator from %q", from)
   243  		return nil
   244  	} else {
   245  		headerBytes = headerBytes[:i+len(headerSep)]
   246  	}
   247  
   248  	ve, err := dkim.ParseAndVerify(string(headerBytes), dkim.HeadersOnly, dnsClient{})
   249  	if err != nil {
   250  		log.Printf("Email from %q didn't pass DKIM verifications: %v", from, err)
   251  		return nil
   252  	}
   253  	if !strings.HasSuffix(ve.Signature.Domain, "google.com") {
   254  		log.Printf("Ignoring DKIM-verified Gerrit email from non-Google domain %q", ve.Signature.Domain)
   255  		return nil
   256  	}
   257  	tp := textproto.NewReader(bufio.NewReader(bytes.NewReader(headerBytes)))
   258  	hdr, err := tp.ReadMIMEHeader()
   259  	if err != nil {
   260  		log.Printf("Ignoring ReadMIMEHeader error: %v    from email:\n%s", err, headerBytes)
   261  		return nil
   262  	}
   263  	if e.from.Hostname() != "gerritcodereview.bounces.google.com" {
   264  		log.Printf("Ignoring signed, DKIM-verified, non-Gerrit email from %q:\n%s", from, bodyBytes)
   265  		return nil
   266  	}
   267  
   268  	changeNum, _ := strconv.Atoi(hdr.Get("X-Gerrit-Change-Number"))
   269  	publish(&pubsubtypes.Event{
   270  		Gerrit: &pubsubtypes.GerritEvent{
   271  			URL:          strings.TrimSuffix(strings.Trim(hdr.Get("X-Gerrit-ChangeURL"), "<>"), "?usp=email"),
   272  			Project:      hdr.Get("X-Gerrit-Project"),
   273  			CommitHash:   hdr.Get("X-Gerrit-Commit"),
   274  			ChangeNumber: changeNum,
   275  		},
   276  	})
   277  	return nil
   278  }
   279  
   280  type eventAndJSON struct {
   281  	*pubsubtypes.Event
   282  	json string // JSON MarshalIndent of Event
   283  }
   284  
   285  var (
   286  	mu      sync.Mutex      // guards following
   287  	recent  []*eventAndJSON // newest at end
   288  	waiting = map[chan *eventAndJSON]struct{}{}
   289  )
   290  
   291  const (
   292  	keepMin = 50
   293  	maxAge  = 1 * time.Hour
   294  )
   295  
   296  func register(ch chan *eventAndJSON, after time.Time) {
   297  	mu.Lock()
   298  	defer mu.Unlock()
   299  	for _, e := range recent {
   300  		if e.Time.Time().After(after) {
   301  			ch <- e
   302  			return
   303  		}
   304  	}
   305  	waiting[ch] = struct{}{}
   306  }
   307  
   308  func unregister(ch chan *eventAndJSON) {
   309  	mu.Lock()
   310  	defer mu.Unlock()
   311  	delete(waiting, ch)
   312  }
   313  
   314  // numOldInRecentLocked returns how many leading items of recent are
   315  // too old.
   316  func numOldInRecentLocked() int {
   317  	if len(recent) <= keepMin {
   318  		return 0
   319  	}
   320  	n := 0
   321  	tooOld := time.Now().Add(-maxAge)
   322  	for _, e := range recent {
   323  		if e.Time.Time().After(tooOld) {
   324  			break
   325  		}
   326  		n++
   327  	}
   328  	return n
   329  }
   330  
   331  func newEventAndJSON(e *pubsubtypes.Event) *eventAndJSON {
   332  	e.Time = types.Time3339(time.Now())
   333  	j, err := json.MarshalIndent(e, "", "\t")
   334  	if err != nil {
   335  		log.Printf("JSON error: %v", err)
   336  	}
   337  	return &eventAndJSON{
   338  		Event: e,
   339  		json:  string(j),
   340  	}
   341  }
   342  
   343  func publish(e *pubsubtypes.Event) {
   344  	ej := newEventAndJSON(e)
   345  	log.Printf("Event: %s", ej.json)
   346  
   347  	mu.Lock()
   348  	defer mu.Unlock()
   349  
   350  	recent = append(recent, ej)
   351  	// Trim old ones off the front of recent
   352  	if n := numOldInRecentLocked(); n > 0 {
   353  		copy(recent, recent[n:])
   354  		recent = recent[:len(recent)-n]
   355  	}
   356  
   357  	for ch := range waiting {
   358  		ch <- ej
   359  		delete(waiting, ch)
   360  	}
   361  }
   362  
   363  type dnsClient struct{}
   364  
   365  var resolver = &net.Resolver{PreferGo: true}
   366  
   367  func (dnsClient) LookupTxt(hostname string) ([]string, error) {
   368  	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
   369  	defer cancel()
   370  	return resolver.LookupTXT(ctx, hostname)
   371  }
   372  
   373  func onNewMail(c smtpd.Connection, from smtpd.MailAddress) (smtpd.Envelope, error) {
   374  	return &env{
   375  		from: from,
   376  		conn: c,
   377  	}, nil
   378  }
   379  
   380  func onNewConnection(c smtpd.Connection) error {
   381  	log.Printf("smtpd: new connection from %v", c.Addr())
   382  	return nil
   383  }