golang.org/x/build@v0.0.0-20240506185731-218518f32b70/cmd/pubsubhelper/pubsubhelper.go (about) 1 // Copyright 2017 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // The pubsubhelper is an SMTP server for Gerrit updates and an HTTP 6 // server for Github webhook updates. It then lets other clients subscribe 7 // to those changes. 8 package main 9 10 import ( 11 "bufio" 12 "bytes" 13 "context" 14 "encoding/json" 15 "errors" 16 "flag" 17 "fmt" 18 "io" 19 "log" 20 "net" 21 "net/http" 22 "net/textproto" 23 "os" 24 "os/signal" 25 "strconv" 26 "strings" 27 "sync" 28 "time" 29 30 "github.com/bradfitz/go-smtpd/smtpd" 31 "github.com/jellevandenhooff/dkim" 32 "go4.org/types" 33 "golang.org/x/build/cmd/pubsubhelper/pubsubtypes" 34 "golang.org/x/build/internal/https" 35 "golang.org/x/build/internal/secret" 36 ) 37 38 var ( 39 botEmail = flag.String("rcpt", "\x67\x6f\x70\x68\x65\x72\x62\x6f\x74@pubsubhelper.golang.org", "email address of bot. incoming emails must be to this address.") 40 smtpListen = flag.String("smtp", ":25", "SMTP listen address") 41 webhookSecret = flag.String("webhook-secret", "", "Development mode GitHub webhook secret. This flag should not be used in production.") 42 ) 43 44 func main() { 45 https.RegisterFlags(flag.CommandLine) 46 flag.Parse() 47 48 ctx := context.Background() 49 ch := make(chan os.Signal, 1) 50 signal.Notify(ch, os.Interrupt) 51 go func() { 52 sig := <-ch 53 log.Printf("Signal %v received; exiting with status 0.", sig) 54 os.Exit(0) 55 }() 56 57 // webhooksecret should not be set in production 58 if *webhookSecret == "" { 59 sc := secret.MustNewClient() 60 defer sc.Close() 61 62 ctxSc, cancel := context.WithTimeout(ctx, 10*time.Second) 63 defer cancel() 64 65 var err error 66 *webhookSecret, err = sc.Retrieve(ctxSc, secret.NamePubSubHelperWebhook) 67 if err != nil { 68 log.Fatalf("unable to retrieve webhook secret %v", err) 69 } 70 } 71 72 http.HandleFunc("/", handleRoot) 73 http.HandleFunc("/waitevent", handleWaitEvent) 74 http.HandleFunc("/recent", handleRecent) 75 http.HandleFunc("/github-webhook", handleGithubWebhook) 76 77 errc := make(chan error) 78 go func() { 79 log.Printf("running pubsubhelper on %s", *smtpListen) 80 s := &smtpd.Server{ 81 Addr: *smtpListen, 82 OnNewMail: onNewMail, 83 OnNewConnection: onNewConnection, 84 ReadTimeout: time.Minute, 85 } 86 err := s.ListenAndServe() 87 errc <- fmt.Errorf("SMTP ListenAndServe: %v", err) 88 }() 89 log.Fatalln(https.ListenAndServe(ctx, http.DefaultServeMux)) 90 } 91 92 func handleRoot(w http.ResponseWriter, r *http.Request) { 93 if r.URL.Path != "/" { 94 http.NotFound(w, r) 95 return 96 } 97 io.WriteString(w, `<html> 98 <body> 99 This is <a href="https://godoc.org/golang.org/x/build/cmd/pubsubhelper">pubsubhelper</a>. 100 101 <ul> 102 <li><b><a href="/waitevent">/waitevent</a></b>: long-poll wait 30s for next event (use ?after=[RFC3339Nano] to resume at point)</li> 103 <li><b><a href="/recent">/recent</a></b>: recent events, without long-polling.</li> 104 </ul> 105 106 </body> 107 </html> 108 `) 109 } 110 111 func handleWaitEvent(w http.ResponseWriter, r *http.Request) { 112 if r.Method != "GET" { 113 http.Error(w, "requires GET", http.StatusBadRequest) 114 return 115 } 116 117 ch := make(chan *eventAndJSON, 1) 118 119 var after time.Time 120 if v := r.FormValue("after"); v != "" { 121 var err error 122 after, err = time.Parse(time.RFC3339Nano, v) 123 if err != nil { 124 http.Error(w, "'after' parameter is not in time.RFC3339Nano format", http.StatusBadRequest) 125 return 126 } 127 } else { 128 after = time.Now() 129 } 130 131 register(ch, after) 132 defer unregister(ch) 133 ctx := r.Context() 134 135 timer := time.NewTimer(30 * time.Second) 136 defer timer.Stop() 137 138 var e *eventAndJSON 139 select { 140 case <-ctx.Done(): 141 return 142 case <-timer.C: 143 e = newEventAndJSON(&pubsubtypes.Event{ 144 LongPollTimeout: true, 145 }) 146 case e = <-ch: 147 } 148 149 w.Header().Set("Content-Type", "application/json; charset=utf-8") 150 io.WriteString(w, e.json) 151 } 152 153 func handleRecent(w http.ResponseWriter, r *http.Request) { 154 w.Header().Set("Content-Type", "application/json; charset=utf-8") 155 156 var after time.Time 157 if v := r.FormValue("after"); v != "" { 158 var err error 159 after, err = time.Parse(time.RFC3339Nano, v) 160 if err != nil { 161 http.Error(w, "'after' parameter is not in time.RFC3339Nano format", http.StatusBadRequest) 162 return 163 } 164 } 165 166 var buf bytes.Buffer 167 mu.Lock() 168 buf.WriteString("[\n") 169 n := 0 170 for i := len(recent) - 1; i >= 0; i-- { 171 ev := recent[i] 172 if ev.Time.Time().Before(after) { 173 continue 174 } 175 if n > 0 { 176 buf.WriteString(",\n") 177 } 178 n++ 179 buf.WriteString(ev.json) 180 } 181 buf.WriteString("\n]\n") 182 mu.Unlock() 183 w.Write(buf.Bytes()) 184 } 185 186 type env struct { 187 from smtpd.MailAddress 188 body bytes.Buffer 189 conn smtpd.Connection 190 tooBig bool 191 hasRcpt bool 192 } 193 194 func (e *env) BeginData() error { 195 if !e.hasRcpt { 196 return smtpd.SMTPError("554 5.5.1 Error: no valid recipients") 197 } 198 return nil 199 } 200 201 func (e *env) AddRecipient(rcpt smtpd.MailAddress) error { 202 if e.hasRcpt { 203 return smtpd.SMTPError("554 5.5.1 Error: dup recipients") 204 } 205 to := rcpt.Email() 206 if to != *botEmail { 207 return errors.New("bogus recipient") 208 } 209 e.hasRcpt = true 210 return nil 211 } 212 213 func (e *env) Write(line []byte) error { 214 const maxSize = 5 << 20 215 if e.body.Len() > maxSize { 216 e.tooBig = true 217 return nil 218 } 219 e.body.Write(line) 220 return nil 221 } 222 223 var ( 224 headerSep = []byte("\r\n\r\n") 225 dkimSignatureHeader = []byte("\nDKIM-Signature:") 226 ) 227 228 func (e *env) Close() error { 229 if e.tooBig { 230 log.Printf("Ignoring too-large email from %q", e.from) 231 return nil 232 } 233 from := e.from.Email() 234 bodyBytes := e.body.Bytes() 235 if !bytes.Contains(bodyBytes, dkimSignatureHeader) { 236 log.Printf("Ignoring unsigned (~spam) email from %q", from) 237 return nil 238 } 239 240 headerBytes := bodyBytes 241 if i := bytes.Index(headerBytes, headerSep); i == -1 { 242 log.Printf("Ignoring email without header separator from %q", from) 243 return nil 244 } else { 245 headerBytes = headerBytes[:i+len(headerSep)] 246 } 247 248 ve, err := dkim.ParseAndVerify(string(headerBytes), dkim.HeadersOnly, dnsClient{}) 249 if err != nil { 250 log.Printf("Email from %q didn't pass DKIM verifications: %v", from, err) 251 return nil 252 } 253 if !strings.HasSuffix(ve.Signature.Domain, "google.com") { 254 log.Printf("Ignoring DKIM-verified Gerrit email from non-Google domain %q", ve.Signature.Domain) 255 return nil 256 } 257 tp := textproto.NewReader(bufio.NewReader(bytes.NewReader(headerBytes))) 258 hdr, err := tp.ReadMIMEHeader() 259 if err != nil { 260 log.Printf("Ignoring ReadMIMEHeader error: %v from email:\n%s", err, headerBytes) 261 return nil 262 } 263 if e.from.Hostname() != "gerritcodereview.bounces.google.com" { 264 log.Printf("Ignoring signed, DKIM-verified, non-Gerrit email from %q:\n%s", from, bodyBytes) 265 return nil 266 } 267 268 changeNum, _ := strconv.Atoi(hdr.Get("X-Gerrit-Change-Number")) 269 publish(&pubsubtypes.Event{ 270 Gerrit: &pubsubtypes.GerritEvent{ 271 URL: strings.TrimSuffix(strings.Trim(hdr.Get("X-Gerrit-ChangeURL"), "<>"), "?usp=email"), 272 Project: hdr.Get("X-Gerrit-Project"), 273 CommitHash: hdr.Get("X-Gerrit-Commit"), 274 ChangeNumber: changeNum, 275 }, 276 }) 277 return nil 278 } 279 280 type eventAndJSON struct { 281 *pubsubtypes.Event 282 json string // JSON MarshalIndent of Event 283 } 284 285 var ( 286 mu sync.Mutex // guards following 287 recent []*eventAndJSON // newest at end 288 waiting = map[chan *eventAndJSON]struct{}{} 289 ) 290 291 const ( 292 keepMin = 50 293 maxAge = 1 * time.Hour 294 ) 295 296 func register(ch chan *eventAndJSON, after time.Time) { 297 mu.Lock() 298 defer mu.Unlock() 299 for _, e := range recent { 300 if e.Time.Time().After(after) { 301 ch <- e 302 return 303 } 304 } 305 waiting[ch] = struct{}{} 306 } 307 308 func unregister(ch chan *eventAndJSON) { 309 mu.Lock() 310 defer mu.Unlock() 311 delete(waiting, ch) 312 } 313 314 // numOldInRecentLocked returns how many leading items of recent are 315 // too old. 316 func numOldInRecentLocked() int { 317 if len(recent) <= keepMin { 318 return 0 319 } 320 n := 0 321 tooOld := time.Now().Add(-maxAge) 322 for _, e := range recent { 323 if e.Time.Time().After(tooOld) { 324 break 325 } 326 n++ 327 } 328 return n 329 } 330 331 func newEventAndJSON(e *pubsubtypes.Event) *eventAndJSON { 332 e.Time = types.Time3339(time.Now()) 333 j, err := json.MarshalIndent(e, "", "\t") 334 if err != nil { 335 log.Printf("JSON error: %v", err) 336 } 337 return &eventAndJSON{ 338 Event: e, 339 json: string(j), 340 } 341 } 342 343 func publish(e *pubsubtypes.Event) { 344 ej := newEventAndJSON(e) 345 log.Printf("Event: %s", ej.json) 346 347 mu.Lock() 348 defer mu.Unlock() 349 350 recent = append(recent, ej) 351 // Trim old ones off the front of recent 352 if n := numOldInRecentLocked(); n > 0 { 353 copy(recent, recent[n:]) 354 recent = recent[:len(recent)-n] 355 } 356 357 for ch := range waiting { 358 ch <- ej 359 delete(waiting, ch) 360 } 361 } 362 363 type dnsClient struct{} 364 365 var resolver = &net.Resolver{PreferGo: true} 366 367 func (dnsClient) LookupTxt(hostname string) ([]string, error) { 368 ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) 369 defer cancel() 370 return resolver.LookupTXT(ctx, hostname) 371 } 372 373 func onNewMail(c smtpd.Connection, from smtpd.MailAddress) (smtpd.Envelope, error) { 374 return &env{ 375 from: from, 376 conn: c, 377 }, nil 378 } 379 380 func onNewConnection(c smtpd.Connection) error { 381 log.Printf("smtpd: new connection from %v", c.Addr()) 382 return nil 383 }