go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/common/api/internal/gensupport/retry.go (about)

     1  // Copyright 2021 Google LLC.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package gensupport
     6  
     7  import (
     8  	"errors"
     9  	"io"
    10  	"net"
    11  	"strings"
    12  	"time"
    13  
    14  	"github.com/googleapis/gax-go/v2"
    15  	"google.golang.org/api/googleapi"
    16  )
    17  
    18  // Backoff is an interface around gax.Backoff's Pause method, allowing tests to provide their
    19  // own implementation.
    20  type Backoff interface {
    21  	Pause() time.Duration
    22  }
    23  
    24  // These are declared as global variables so that tests can overwrite them.
    25  var (
    26  	// Default per-chunk deadline for resumable uploads.
    27  	defaultRetryDeadline = 32 * time.Second
    28  	// Default backoff timer.
    29  	backoff = func() Backoff {
    30  		return &gax.Backoff{Initial: 100 * time.Millisecond}
    31  	}
    32  	// syscallRetryable is a platform-specific hook, specified in retryable_linux.go
    33  	syscallRetryable func(error) bool = func(err error) bool { return false }
    34  )
    35  
    36  const (
    37  	// statusTooManyRequests is returned by the storage API if the
    38  	// per-project limits have been temporarily exceeded. The request
    39  	// should be retried.
    40  	// https://cloud.google.com/storage/docs/json_api/v1/status-codes#standardcodes
    41  	statusTooManyRequests = 429
    42  
    43  	// statusRequestTimeout is returned by the storage API if the
    44  	// upload connection was broken. The request should be retried.
    45  	statusRequestTimeout = 408
    46  )
    47  
    48  // shouldRetry indicates whether an error is retryable for the purposes of this
    49  // package, unless a ShouldRetry func is specified by the RetryConfig instead.
    50  // It follows guidance from
    51  // https://cloud.google.com/storage/docs/exponential-backoff .
    52  func shouldRetry(status int, err error) bool {
    53  	if 500 <= status && status <= 599 {
    54  		return true
    55  	}
    56  	if status == statusTooManyRequests || status == statusRequestTimeout {
    57  		return true
    58  	}
    59  	if err == io.ErrUnexpectedEOF {
    60  		return true
    61  	}
    62  	// Transient network errors should be retried.
    63  	if syscallRetryable(err) {
    64  		return true
    65  	}
    66  	if err, ok := err.(interface{ Temporary() bool }); ok {
    67  		if err.Temporary() {
    68  			return true
    69  		}
    70  	}
    71  	var opErr *net.OpError
    72  	if errors.As(err, &opErr) {
    73  		if strings.Contains(opErr.Error(), "use of closed network connection") {
    74  			// TODO: check against net.ErrClosed (go 1.16+) instead of string
    75  			return true
    76  		}
    77  	}
    78  
    79  	// If Go 1.13 error unwrapping is available, use this to examine wrapped
    80  	// errors.
    81  	if err, ok := err.(interface{ Unwrap() error }); ok {
    82  		return shouldRetry(status, err.Unwrap())
    83  	}
    84  	return false
    85  }
    86  
    87  // RetryConfig allows configuration of backoff timing and retryable errors.
    88  type RetryConfig struct {
    89  	Backoff     *gax.Backoff
    90  	ShouldRetry func(err error) bool
    91  }
    92  
    93  // Get a new backoff object based on the configured values.
    94  func (r *RetryConfig) backoff() Backoff {
    95  	if r == nil || r.Backoff == nil {
    96  		return backoff()
    97  	}
    98  	return &gax.Backoff{
    99  		Initial:    r.Backoff.Initial,
   100  		Max:        r.Backoff.Max,
   101  		Multiplier: r.Backoff.Multiplier,
   102  	}
   103  }
   104  
   105  // This is kind of hacky; it is necessary because ShouldRetry expects to
   106  // handle HTTP errors via googleapi.Error, but the error has not yet been
   107  // wrapped with a googleapi.Error at this layer, and the ErrorFunc type
   108  // in the manual layer does not pass in a status explicitly as it does
   109  // here. So, we must wrap error status codes in a googleapi.Error so that
   110  // ShouldRetry can parse this correctly.
   111  func (r *RetryConfig) errorFunc() func(status int, err error) bool {
   112  	if r == nil || r.ShouldRetry == nil {
   113  		return shouldRetry
   114  	}
   115  	return func(status int, err error) bool {
   116  		if status >= 400 {
   117  			return r.ShouldRetry(&googleapi.Error{Code: status})
   118  		}
   119  		return r.ShouldRetry(err)
   120  	}
   121  }