golang.org/x/build@v0.0.0-20240506185731-218518f32b70/internal/task/announce.go (about)

     1  // Copyright 2021 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package task
     6  
     7  import (
     8  	"bytes"
     9  	"embed"
    10  	"errors"
    11  	"fmt"
    12  	"io"
    13  	"mime"
    14  	"net/http"
    15  	"net/mail"
    16  	"net/url"
    17  	"strings"
    18  	"text/template"
    19  	"time"
    20  
    21  	sendgrid "github.com/sendgrid/sendgrid-go"
    22  	sendgridmail "github.com/sendgrid/sendgrid-go/helpers/mail"
    23  	"github.com/yuin/goldmark"
    24  	"github.com/yuin/goldmark/ast"
    25  	"github.com/yuin/goldmark/extension"
    26  	"github.com/yuin/goldmark/renderer"
    27  	goldmarkhtml "github.com/yuin/goldmark/renderer/html"
    28  	goldmarktext "github.com/yuin/goldmark/text"
    29  	"golang.org/x/build/internal/gophers"
    30  	"golang.org/x/build/internal/workflow"
    31  	"golang.org/x/build/maintner/maintnerd/maintapi/version"
    32  	"golang.org/x/net/html"
    33  )
    34  
    35  type releaseAnnouncement struct {
    36  	// Kind is the kind of release being announced.
    37  	Kind ReleaseKind
    38  
    39  	// Version is the Go version that has been released.
    40  	//
    41  	// The version string must use the same format as Go tags. For example:
    42  	//   - "go1.21rc2" for a pre-release
    43  	//   - "go1.21.0" for a major Go release
    44  	//   - "go1.21.1" for a minor Go release
    45  	Version string
    46  	// SecondaryVersion is an older Go version that was also released.
    47  	// This only applies to minor releases when two releases are made.
    48  	// For example, "go1.20.9".
    49  	SecondaryVersion string
    50  
    51  	// Security is a list of descriptions, one for each distinct
    52  	// security fix included in this release, in Markdown format.
    53  	//
    54  	// The empty list means there are no security fixes included.
    55  	//
    56  	// This field applies only to minor releases; it is an error
    57  	// to try to use it another release type.
    58  	Security []string
    59  
    60  	// Names is an optional list of release coordinator names to
    61  	// include in the sign-off message.
    62  	Names []string
    63  }
    64  
    65  type releasePreAnnouncement struct {
    66  	// Target is the planned date for the release.
    67  	Target Date
    68  
    69  	// Version is the Go version that will be released.
    70  	//
    71  	// The version string must use the same format as Go tags. For example, "go1.17.2".
    72  	Version string
    73  	// SecondaryVersion is an older Go version that will also be released.
    74  	// This only applies when two releases are planned. For example, "go1.16.10".
    75  	SecondaryVersion string
    76  
    77  	// Security is the security content to be included in the
    78  	// release pre-announcement. It should not reveal details
    79  	// beyond what's allowed by the security policy.
    80  	Security string
    81  
    82  	// CVEs is the list of CVEs for PRIVATE track fixes to
    83  	// be included in the release pre-announcement.
    84  	CVEs []string
    85  
    86  	// Names is an optional list of release coordinator names to
    87  	// include in the sign-off message.
    88  	Names []string
    89  }
    90  
    91  // A Date represents a single calendar day (year, month, day).
    92  //
    93  // This type does not include location information, and
    94  // therefore does not describe a unique 24-hour timespan.
    95  //
    96  // TODO(go.dev/issue/19700): Start using time.Day or so when available.
    97  type Date struct {
    98  	Year  int        // Year (for example, 2009).
    99  	Month time.Month // Month of the year (January = 1, ...).
   100  	Day   int        // Day of the month, starting at 1.
   101  }
   102  
   103  func (d Date) String() string { return d.Format("2006-01-02") }
   104  func (d Date) Format(layout string) string {
   105  	return time.Date(d.Year, d.Month, d.Day, 0, 0, 0, 0, time.UTC).Format(layout)
   106  }
   107  func (d Date) After(year int, month time.Month, day int) bool {
   108  	return time.Date(d.Year, d.Month, d.Day, 0, 0, 0, 0, time.UTC).
   109  		After(time.Date(year, month, day, 0, 0, 0, 0, time.UTC))
   110  }
   111  
   112  // AnnounceMailTasks contains tasks related to the release (pre-)announcement email.
   113  type AnnounceMailTasks struct {
   114  	// SendMail sends an email with the given header and content
   115  	// using an externally-provided implementation.
   116  	//
   117  	// Email delivery happens asynchronously, so SendMail returns a nil error
   118  	// if the transmission was started successfully, but that error value
   119  	// doesn't indicate anything about the status of the delivery.
   120  	SendMail func(MailHeader, MailContent) error
   121  
   122  	// AnnounceMailHeader is the header to use for the release (pre-)announcement email.
   123  	AnnounceMailHeader MailHeader
   124  
   125  	// testHookNow is optionally set by tests to override time.Now.
   126  	testHookNow func() time.Time
   127  }
   128  
   129  // SentMail represents an email that was sent.
   130  type SentMail struct {
   131  	Subject string // Subject of the email. Expected to be unique so it can be used to identify the email.
   132  }
   133  
   134  // AnnounceRelease sends an email to Google Groups
   135  // announcing that a Go release has been published.
   136  func (t AnnounceMailTasks) AnnounceRelease(ctx *workflow.TaskContext, kind ReleaseKind, published []Published, security []string, users []string) (SentMail, error) {
   137  	if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < time.Minute {
   138  		return SentMail{}, fmt.Errorf("insufficient time for announce release task; a minimum of a minute left on context is required")
   139  	}
   140  	if len(published) < 1 || len(published) > 2 {
   141  		return SentMail{}, fmt.Errorf("got %d published Go releases, AnnounceRelease supports only 1 or 2 at once", len(published))
   142  	}
   143  	names, err := coordinatorFirstNames(users)
   144  	if err != nil {
   145  		return SentMail{}, err
   146  	}
   147  
   148  	r := releaseAnnouncement{
   149  		Kind:     kind,
   150  		Version:  published[0].Version,
   151  		Security: security,
   152  		Names:    names,
   153  	}
   154  	if len(published) == 2 {
   155  		r.SecondaryVersion = published[1].Version
   156  	}
   157  
   158  	// Generate the announcement email.
   159  	m, err := announcementMail(r)
   160  	if err != nil {
   161  		return SentMail{}, err
   162  	}
   163  	ctx.Printf("announcement subject: %s\n\n", m.Subject)
   164  	ctx.Printf("announcement body HTML:\n%s\n", m.BodyHTML)
   165  	ctx.Printf("announcement body text:\n%s", m.BodyText)
   166  
   167  	// Before sending, check to see if this announcement already exists.
   168  	if threadURL, err := findGoogleGroupsThread(ctx, m.Subject); err != nil {
   169  		// Proceeding would risk sending a duplicate email, so error out instead.
   170  		return SentMail{}, fmt.Errorf("stopping early due to error checking for an existing Google Groups thread: %w", err)
   171  	} else if threadURL != "" {
   172  		// This should never happen since this task runs once per release.
   173  		// It can happen under unusual circumstances, for example if the task crashes after
   174  		// mailing but before completion, or if parts of the release workflow are restarted,
   175  		// or if a human mails the announcement email manually out of band.
   176  		//
   177  		// So if we see that the email exists, consider it as "task completed successfully"
   178  		// and pretend we were the ones that sent it, so the high level workflow can keep going.
   179  		ctx.Printf("a Google Groups thread with matching subject %q already exists at %q, so we'll consider that as it being sent successfully", m.Subject, threadURL)
   180  		return SentMail{m.Subject}, nil
   181  	}
   182  
   183  	// Send the announcement email to the destination mailing lists.
   184  	if t.SendMail == nil {
   185  		return SentMail{Subject: "[dry-run] " + m.Subject}, nil
   186  	}
   187  	ctx.DisableRetries()
   188  	err = t.SendMail(t.AnnounceMailHeader, m)
   189  	if err != nil {
   190  		return SentMail{}, err
   191  	}
   192  
   193  	return SentMail{m.Subject}, nil
   194  }
   195  
   196  // PreAnnounceRelease sends an email pre-announcing a Go release
   197  // containing PRIVATE track security fixes planned for the target date.
   198  func (t AnnounceMailTasks) PreAnnounceRelease(ctx *workflow.TaskContext, versions []string, target Date, security string, cves []string, users []string) (SentMail, error) {
   199  	if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < time.Minute {
   200  		return SentMail{}, fmt.Errorf("insufficient time for pre-announce release task; a minimum of a minute left on context is required")
   201  	}
   202  	if len(versions) < 1 || len(versions) > 2 {
   203  		return SentMail{}, fmt.Errorf("got %d planned Go releases, PreAnnounceRelease supports only 1 or 2 at once", len(versions))
   204  	}
   205  	now := time.Now().UTC()
   206  	if t.testHookNow != nil {
   207  		now = t.testHookNow()
   208  	}
   209  	if !target.After(now.Year(), now.Month(), now.Day()) { // A very simple check. Improve as needed.
   210  		return SentMail{}, fmt.Errorf("target release date is not in the future")
   211  	}
   212  	if security == "" {
   213  		return SentMail{}, fmt.Errorf("security content is not specified")
   214  	}
   215  	if len(cves) == 0 {
   216  		return SentMail{}, errors.New("CVEs are not specified")
   217  	}
   218  	names, err := coordinatorFirstNames(users)
   219  	if err != nil {
   220  		return SentMail{}, err
   221  	}
   222  
   223  	r := releasePreAnnouncement{
   224  		Target:   target,
   225  		Version:  versions[0],
   226  		Security: security,
   227  		CVEs:     cves,
   228  		Names:    names,
   229  	}
   230  	if len(versions) == 2 {
   231  		r.SecondaryVersion = versions[1]
   232  	}
   233  
   234  	// Generate the pre-announcement email.
   235  	m, err := announcementMail(r)
   236  	if err != nil {
   237  		return SentMail{}, err
   238  	}
   239  	ctx.Printf("pre-announcement subject: %s\n\n", m.Subject)
   240  	ctx.Printf("pre-announcement body HTML:\n%s\n", m.BodyHTML)
   241  	ctx.Printf("pre-announcement body text:\n%s", m.BodyText)
   242  
   243  	// Before sending, check to see if this pre-announcement already exists.
   244  	if threadURL, err := findGoogleGroupsThread(ctx, m.Subject); err != nil {
   245  		return SentMail{}, fmt.Errorf("stopping early due to error checking for an existing Google Groups thread: %w", err)
   246  	} else if threadURL != "" {
   247  		ctx.Printf("a Google Groups thread with matching subject %q already exists at %q, so we'll consider that as it being sent successfully", m.Subject, threadURL)
   248  		return SentMail{m.Subject}, nil
   249  	}
   250  
   251  	// Send the pre-announcement email to the destination mailing lists.
   252  	if t.SendMail == nil {
   253  		return SentMail{Subject: "[dry-run] " + m.Subject}, nil
   254  	}
   255  	ctx.DisableRetries()
   256  	err = t.SendMail(t.AnnounceMailHeader, m)
   257  	if err != nil {
   258  		return SentMail{}, err
   259  	}
   260  
   261  	return SentMail{m.Subject}, nil
   262  }
   263  
   264  func coordinatorFirstNames(users []string) ([]string, error) {
   265  	return mapCoordinators(users, func(p *gophers.Person) string {
   266  		name, _, _ := strings.Cut(p.Name, " ")
   267  		return name
   268  	})
   269  }
   270  
   271  func coordinatorEmails(users []string) ([]string, error) {
   272  	return mapCoordinators(users, func(p *gophers.Person) string {
   273  		return p.Gerrit
   274  	})
   275  }
   276  
   277  func mapCoordinators(users []string, f func(*gophers.Person) string) ([]string, error) {
   278  	var outs []string
   279  	for _, user := range users {
   280  		person, err := lookupCoordinator(user)
   281  		if err != nil {
   282  			return nil, err
   283  		}
   284  		outs = append(outs, f(person))
   285  	}
   286  	return outs, nil
   287  }
   288  
   289  // CheckCoordinators checks that all users are known
   290  // and have required information (name, Gerrit email).
   291  func CheckCoordinators(users []string) error {
   292  	var report strings.Builder
   293  	for _, user := range users {
   294  		if _, err := lookupCoordinator(user); err != nil {
   295  			fmt.Fprintln(&report, err)
   296  		}
   297  	}
   298  	if report.Len() != 0 {
   299  		return errors.New(report.String())
   300  	}
   301  	return nil
   302  }
   303  
   304  func lookupCoordinator(user string) (*gophers.Person, error) {
   305  	person := gophers.GetPerson(user + "@golang.org")
   306  	if person == nil {
   307  		person = gophers.GetPerson(user + "@google.com")
   308  	}
   309  	if person == nil {
   310  		return nil, fmt.Errorf("unknown username %q: no @golang or @google account", user)
   311  	} else if person.Name == "" {
   312  		return nil, fmt.Errorf("release coordinator %q is missing a name", person.Gerrit)
   313  	}
   314  	return person, nil
   315  }
   316  
   317  // A MailHeader is an email header.
   318  type MailHeader struct {
   319  	From mail.Address // An RFC 5322 address. For example, "Barry Gibbs <bg@example.com>".
   320  	To   mail.Address
   321  	BCC  []mail.Address
   322  }
   323  
   324  // A MailContent holds the content of an email.
   325  type MailContent struct {
   326  	Subject  string
   327  	BodyHTML string
   328  	BodyText string
   329  }
   330  
   331  // announcementMail generates the (pre-)announcement email using data,
   332  // which must be one of these types:
   333  //   - releaseAnnouncement for a release announcement
   334  //   - releasePreAnnouncement for a release pre-announcement
   335  func announcementMail(data any) (MailContent, error) {
   336  	// Select the appropriate template name.
   337  	var name string
   338  	switch r := data.(type) {
   339  	case releaseAnnouncement:
   340  		switch r.Kind {
   341  		case KindBeta:
   342  			name = "announce-beta.md"
   343  		case KindRC:
   344  			name = "announce-rc.md"
   345  		case KindMajor:
   346  			name = "announce-major.md"
   347  		case KindMinor:
   348  			name = "announce-minor.md"
   349  		default:
   350  			return MailContent{}, fmt.Errorf("unknown release kind: %v", r.Kind)
   351  		}
   352  		if len(r.Security) > 0 && name != "announce-minor.md" {
   353  			// The Security field isn't supported in templates other than minor,
   354  			// so report an error instead of silently dropping it.
   355  			//
   356  			// Note: Maybe in the future we'd want to consider support for including sentences like
   357  			// "This beta release includes the same security fixes as in Go X.Y.Z and Go A.B.C.",
   358  			// but we'll have a better idea after these initial templates get more practical use.
   359  			return MailContent{}, fmt.Errorf("email template %q doesn't support the Security field; this field can only be used in minor releases", name)
   360  		} else if r.SecondaryVersion != "" && name != "announce-minor.md" {
   361  			return MailContent{}, fmt.Errorf("email template %q doesn't support more than one release; the SecondaryVersion field can only be used in minor releases", name)
   362  		}
   363  	case releasePreAnnouncement:
   364  		name = "pre-announce-minor.md"
   365  	default:
   366  		return MailContent{}, fmt.Errorf("unknown template data type %T", data)
   367  	}
   368  
   369  	// Render the (pre-)announcement email template.
   370  	//
   371  	// It'll produce a valid message with a MIME header and a body, so parse it as such.
   372  	var buf bytes.Buffer
   373  	if err := announceTmpl.ExecuteTemplate(&buf, name, data); err != nil {
   374  		return MailContent{}, err
   375  	}
   376  	m, err := mail.ReadMessage(&buf)
   377  	if err != nil {
   378  		return MailContent{}, fmt.Errorf(`email template must be formatted like a mail message, but reading it failed: %v`, err)
   379  	}
   380  
   381  	// Get the email subject (it's a plain string, no further processing needed).
   382  	if _, ok := m.Header["Subject"]; !ok {
   383  		return MailContent{}, fmt.Errorf(`email template must have a "Subject" key in its MIME header, but it's not found`)
   384  	} else if n := len(m.Header["Subject"]); n != 1 {
   385  		return MailContent{}, fmt.Errorf(`email template must have a single "Subject" value in its MIME header, but have %d values`, n)
   386  	}
   387  	subject := m.Header.Get("Subject")
   388  
   389  	// Render the email body, in Markdown format at this point, to HTML and plain text.
   390  	html, text, err := renderMarkdown(m.Body)
   391  	if err != nil {
   392  		return MailContent{}, err
   393  	}
   394  
   395  	return MailContent{subject, html, text}, nil
   396  }
   397  
   398  // announceTmpl holds templates for Go release announcement emails.
   399  //
   400  // Each email template starts with a MIME-style header with a Subject key,
   401  // and the rest of it is Markdown for the email body.
   402  var announceTmpl = template.Must(template.New("").Funcs(template.FuncMap{
   403  	"join": func(s []string) string {
   404  		switch len(s) {
   405  		case 0:
   406  			return ""
   407  		case 1:
   408  			return s[0]
   409  		case 2:
   410  			return s[0] + " and " + s[1]
   411  		default:
   412  			return strings.Join(s[:len(s)-1], ", ") + ", and " + s[len(s)-1]
   413  		}
   414  	},
   415  	"indent": func(s string) string { return "\t" + strings.ReplaceAll(s, "\n", "\n\t") },
   416  
   417  	// subjectPrefix returns the email subject prefix for release r, if any.
   418  	"subjectPrefix": func(r releaseAnnouncement) string {
   419  		switch {
   420  		case len(r.Security) > 0:
   421  			// Include a security prefix as documented at https://go.dev/security#receiving-security-updates:
   422  			//
   423  			//	> The best way to receive security announcements is to subscribe to the golang-announce mailing list.
   424  			//	> Any messages pertaining to a security issue will be prefixed with [security].
   425  			//
   426  			return "[security]"
   427  		default:
   428  			return ""
   429  		}
   430  	},
   431  
   432  	// short and helpers below manipulate valid Go version strings
   433  	// for the current needs of the announcement templates.
   434  	"short": func(v string) string { return strings.TrimPrefix(v, "go") },
   435  	// major extracts the major prefix of a valid Go version.
   436  	// For example, major("go1.18.4") == "1.18".
   437  	"major": func(v string) (string, error) {
   438  		x, ok := version.Go1PointX(v)
   439  		if !ok {
   440  			return "", fmt.Errorf("internal error: version.Go1PointX(%q) is not ok", v)
   441  		}
   442  		return fmt.Sprintf("1.%d", x), nil
   443  	},
   444  	// build extracts the pre-release build number of a valid Go version.
   445  	// For example, build("go1.19beta2") == "2".
   446  	"build": func(v string) (string, error) {
   447  		if i := strings.Index(v, "beta"); i != -1 {
   448  			return v[i+len("beta"):], nil
   449  		} else if i := strings.Index(v, "rc"); i != -1 {
   450  			return v[i+len("rc"):], nil
   451  		}
   452  		return "", fmt.Errorf("internal error: unhandled pre-release Go version %q", v)
   453  	},
   454  }).ParseFS(tmplDir, "template/announce-*.md", "template/pre-announce-minor.md"))
   455  
   456  //go:embed template
   457  var tmplDir embed.FS
   458  
   459  type realSendGridMailClient struct {
   460  	sg *sendgrid.Client
   461  }
   462  
   463  // NewSendGridMailClient creates a SendGrid mail client
   464  // authenticated with the given API key.
   465  func NewSendGridMailClient(sendgridAPIKey string) realSendGridMailClient {
   466  	return realSendGridMailClient{sg: sendgrid.NewSendClient(sendgridAPIKey)}
   467  }
   468  
   469  // SendMail sends an email by making an authenticated request to the SendGrid API.
   470  func (c realSendGridMailClient) SendMail(h MailHeader, m MailContent) error {
   471  	from, to := sendgridmail.Email(h.From), sendgridmail.Email(h.To)
   472  	req := sendgridmail.NewSingleEmail(&from, m.Subject, &to, m.BodyText, m.BodyHTML)
   473  	if len(req.Personalizations) != 1 {
   474  		return fmt.Errorf("internal error: len(req.Personalizations) is %d, want 1", len(req.Personalizations))
   475  	}
   476  	for _, bcc := range h.BCC {
   477  		bcc := sendgridmail.Email(bcc)
   478  		req.Personalizations[0].AddBCCs(&bcc)
   479  	}
   480  	no := false
   481  	req.TrackingSettings = &sendgridmail.TrackingSettings{
   482  		ClickTracking:        &sendgridmail.ClickTrackingSetting{Enable: &no},
   483  		OpenTracking:         &sendgridmail.OpenTrackingSetting{Enable: &no},
   484  		SubscriptionTracking: &sendgridmail.SubscriptionTrackingSetting{Enable: &no},
   485  	}
   486  	resp, err := c.sg.Send(req)
   487  	if err != nil {
   488  		return err
   489  	} else if resp.StatusCode != http.StatusAccepted {
   490  		return fmt.Errorf("unexpected status %d %s, want 202 Accepted; body = %s", resp.StatusCode, http.StatusText(resp.StatusCode), resp.Body)
   491  	}
   492  	return nil
   493  }
   494  
   495  // AwaitAnnounceMail waits for an announcement email with the specified subject
   496  // to show up on Google Groups, and returns its canonical URL.
   497  func (t AnnounceMailTasks) AwaitAnnounceMail(ctx *workflow.TaskContext, m SentMail) (announcementURL string, _ error) {
   498  	// Find the URL for the announcement while giving the email a chance to be received and moderated.
   499  	check := func() (string, bool, error) {
   500  		// See if our email is available by now.
   501  		threadURL, err := findGoogleGroupsThread(ctx, m.Subject)
   502  		if err != nil {
   503  			ctx.Printf("findGoogleGroupsThread: %v", err)
   504  			return "", false, nil
   505  		}
   506  		return threadURL, threadURL != "", nil
   507  
   508  	}
   509  	return AwaitCondition(ctx, 10*time.Second, check)
   510  }
   511  
   512  // findGoogleGroupsThread fetches the first page of threads from the golang-announce
   513  // Google Groups mailing list, and looks for a thread with the matching subject line.
   514  // It returns its URL if found or the empty string if not found.
   515  //
   516  // findGoogleGroupsThread returns an error that matches fetchError with
   517  // PossiblyRetryable set to true when it has signal that repeating the
   518  // same call after some time may succeed.
   519  func findGoogleGroupsThread(ctx *workflow.TaskContext, subject string) (threadURL string, _ error) {
   520  	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://groups.google.com/g/golang-announce", nil)
   521  	if err != nil {
   522  		return "", err
   523  	}
   524  	resp, err := http.DefaultClient.Do(req)
   525  	if err != nil {
   526  		return "", fetchError{Err: err, PossiblyRetryable: true}
   527  	}
   528  	defer resp.Body.Close()
   529  	if resp.StatusCode != http.StatusOK {
   530  		possiblyRetryable := resp.StatusCode/100 == 5 // Consider a 5xx server response to possibly succeed later.
   531  		body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
   532  		return "", fetchError{fmt.Errorf("did not get acceptable status code: %v body: %q", resp.Status, body), possiblyRetryable}
   533  	}
   534  	if ct, want := resp.Header.Get("Content-Type"), "text/html; charset=utf-8"; ct != want {
   535  		ctx.Printf("findGoogleGroupsThread: got response with non-'text/html; charset=utf-8' Content-Type header %q\n", ct)
   536  		if mediaType, _, err := mime.ParseMediaType(ct); err != nil {
   537  			return "", fmt.Errorf("bad Content-Type header %q: %v", ct, err)
   538  		} else if mediaType != "text/html" {
   539  			return "", fmt.Errorf("got media type %q, want %q", mediaType, "text/html")
   540  		}
   541  	}
   542  	doc, err := html.Parse(retryableReader{io.LimitReader(resp.Body, 5<<20)})
   543  	if err != nil {
   544  		return "", err
   545  	}
   546  	var baseHref string
   547  	var linkHref string
   548  	var found bool
   549  	var f func(*html.Node)
   550  	f = func(n *html.Node) {
   551  		if n.Type == html.ElementNode && n.Data == "base" {
   552  			baseHref = href(n)
   553  		} else if n.Type == html.ElementNode && n.Data == "a" {
   554  			linkHref = href(n)
   555  		} else if n.Type == html.TextNode && n.Data == subject {
   556  			// Found our link. Break out.
   557  			found = true
   558  			return
   559  		}
   560  		for c := n.FirstChild; c != nil && !found; c = c.NextSibling {
   561  			f(c)
   562  		}
   563  	}
   564  	f(doc)
   565  	if !found {
   566  		// Thread not found on the first page.
   567  		return "", nil
   568  	}
   569  	base, err := url.Parse(baseHref)
   570  	if err != nil {
   571  		return "", err
   572  	}
   573  	link, err := url.Parse(linkHref)
   574  	if err != nil {
   575  		return "", err
   576  	}
   577  	threadURL = base.ResolveReference(link).String()
   578  	const announcementPrefix = "https://groups.google.com/g/golang-announce/c/"
   579  	if !strings.HasPrefix(threadURL, announcementPrefix) {
   580  		return "", fmt.Errorf("found URL %q, but it doesn't have the expected prefix %q", threadURL, announcementPrefix)
   581  	}
   582  	return threadURL, nil
   583  }
   584  
   585  func href(n *html.Node) string {
   586  	for _, a := range n.Attr {
   587  		if a.Key == "href" {
   588  			return a.Val
   589  		}
   590  	}
   591  	return ""
   592  }
   593  
   594  // retryableReader annotates errors from
   595  // RetryableReader as possibly retryable.
   596  type retryableReader struct{ RetryableReader io.Reader }
   597  
   598  func (r retryableReader) Read(p []byte) (n int, err error) {
   599  	n, err = r.RetryableReader.Read(p)
   600  	if err != nil && err != io.EOF {
   601  		err = fetchError{Err: err, PossiblyRetryable: true}
   602  	}
   603  	return n, err
   604  }
   605  
   606  // fetchError records an error during a fetch operation over an unreliable network.
   607  type fetchError struct {
   608  	Err error // Non-nil.
   609  
   610  	// PossiblyRetryable indicates whether Err is believed to be possibly caused by a
   611  	// non-terminal network error, such that the caller can expect it may not happen
   612  	// again if it simply tries the same fetch operation again after waiting a bit.
   613  	PossiblyRetryable bool
   614  }
   615  
   616  func (e fetchError) Error() string { return e.Err.Error() }
   617  func (e fetchError) Unwrap() error { return e.Err }
   618  
   619  // renderMarkdown parses Markdown source
   620  // and renders it to HTML and plain text.
   621  //
   622  // The Markdown specification and its various extensions are vast.
   623  // Here we support a small, simple set of Markdown features
   624  // that satisfies the needs of the announcement mail tasks.
   625  func renderMarkdown(r io.Reader) (html, text string, _ error) {
   626  	source, err := io.ReadAll(r)
   627  	if err != nil {
   628  		return "", "", err
   629  	}
   630  	// Configure a Markdown parser and HTML renderer fairly closely
   631  	// to how it's done in x/website, just without raw HTML support
   632  	// and other extensions we don't need for announcement emails.
   633  	md := goldmark.New(
   634  		goldmark.WithRendererOptions(goldmarkhtml.WithHardWraps()),
   635  		goldmark.WithExtensions(
   636  			extension.NewLinkify(extension.WithLinkifyAllowedProtocols([][]byte{[]byte("https:")})),
   637  		),
   638  	)
   639  	doc := md.Parser().Parse(goldmarktext.NewReader(source))
   640  	var htmlBuf, textBuf bytes.Buffer
   641  	err = md.Renderer().Render(&htmlBuf, source, doc)
   642  	if err != nil {
   643  		return "", "", err
   644  	}
   645  	err = (markdownToTextRenderer{}).Render(&textBuf, source, doc)
   646  	if err != nil {
   647  		return "", "", err
   648  	}
   649  	return htmlBuf.String(), textBuf.String(), nil
   650  }
   651  
   652  // markdownToTextRenderer is a simple goldmark/renderer.Renderer implementation
   653  // that renders Markdown to plain text for the needs of announcement mail tasks.
   654  //
   655  // It produces an output suitable for email clients that cannot (or choose not to)
   656  // display the HTML version of the email. (It also helps a bit with the readability
   657  // of our test data, since without a browser plain text is more readable than HTML.)
   658  //
   659  // The output is mostly plain text that doesn't preserve Markdown syntax (for example,
   660  // `code` is rendered without backticks), though there is very lightweight formatting
   661  // applied (links are written as "text <URL>").
   662  //
   663  // We can in theory choose to delete this renderer at any time if its maintenance costs
   664  // start to outweight its benefits, since Markdown by definition is designed to be human
   665  // readable and can be used as plain text without any processing.
   666  type markdownToTextRenderer struct{}
   667  
   668  func (markdownToTextRenderer) Render(w io.Writer, source []byte, n ast.Node) error {
   669  	const debug = false
   670  	if debug {
   671  		n.Dump(source, 0)
   672  	}
   673  
   674  	var (
   675  		markers []byte // Stack of list markers, from outermost to innermost.
   676  	)
   677  	walk := func(n ast.Node, entering bool) (ast.WalkStatus, error) {
   678  		if entering {
   679  			if n.Type() == ast.TypeBlock && n.PreviousSibling() != nil {
   680  				// Print a blank line between block nodes.
   681  				switch n.PreviousSibling().Kind() {
   682  				default:
   683  					fmt.Fprint(w, "\n\n")
   684  				case ast.KindCodeBlock:
   685  					// A code block always ends with a newline, so only need one more.
   686  					fmt.Fprintln(w)
   687  				}
   688  
   689  				// If we're in a list, indent accordingly.
   690  				if n.Kind() != ast.KindListItem {
   691  					fmt.Fprint(w, strings.Repeat("\t", len(markers)))
   692  				}
   693  			}
   694  
   695  			switch n := n.(type) {
   696  			case *ast.Text:
   697  				fmt.Fprintf(w, "%s", n.Text(source))
   698  
   699  				// Print a line break.
   700  				if n.SoftLineBreak() || n.HardLineBreak() {
   701  					fmt.Fprintln(w)
   702  
   703  					// If we're in a list, indent accordingly.
   704  					fmt.Fprint(w, strings.Repeat("\t", len(markers)))
   705  				}
   706  			case *ast.CodeBlock:
   707  				indent := strings.Repeat("\t", len(markers)+1) // Indent if in a list, plus one more since it's a code block.
   708  				for i := 0; i < n.Lines().Len(); i++ {
   709  					s := n.Lines().At(i)
   710  					fmt.Fprint(w, indent, string(source[s.Start:s.Stop]))
   711  				}
   712  			case *ast.AutoLink:
   713  				// Auto-links are printed as is in plain text.
   714  				//
   715  				// For example, the Markdown "https://go.dev/issue/123"
   716  				// is rendered as plain text "https://go.dev/issue/123".
   717  				fmt.Fprint(w, string(n.Label(source)))
   718  			case *ast.List:
   719  				// Push list marker on the stack.
   720  				markers = append(markers, n.Marker)
   721  			case *ast.ListItem:
   722  				fmt.Fprintf(w, "%s%c\t", strings.Repeat("\t", len(markers)-1), markers[len(markers)-1])
   723  			}
   724  		} else {
   725  			switch n := n.(type) {
   726  			case *ast.Link:
   727  				// Append the link's URL after its text.
   728  				//
   729  				// For example, the Markdown "[security policy](https://go.dev/security)"
   730  				// is rendered as plain text "security policy <https://go.dev/security>".
   731  				fmt.Fprintf(w, " <%s>", n.Destination)
   732  			case *ast.List:
   733  				// Pop list marker off the stack.
   734  				markers = markers[:len(markers)-1]
   735  			}
   736  
   737  			if n.Type() == ast.TypeDocument && n.ChildCount() != 0 {
   738  				// Print a newline at the end of the document, if it's not empty.
   739  				fmt.Fprintln(w)
   740  			}
   741  		}
   742  		return ast.WalkContinue, nil
   743  	}
   744  	return ast.Walk(n, walk)
   745  }
   746  func (markdownToTextRenderer) AddOptions(...renderer.Option) {}