storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/rest/client.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2018-2020 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package rest
    18  
    19  import (
    20  	"context"
    21  	"errors"
    22  	"fmt"
    23  	"io"
    24  	"io/ioutil"
    25  	"math/rand"
    26  	"net/http"
    27  	"net/url"
    28  	"sync/atomic"
    29  	"time"
    30  
    31  	xhttp "storj.io/minio/cmd/http"
    32  	"storj.io/minio/cmd/logger"
    33  	xnet "storj.io/minio/pkg/net"
    34  )
    35  
    36  // DefaultTimeout - default REST timeout is 10 seconds.
    37  const DefaultTimeout = 10 * time.Second
    38  
    39  const (
    40  	offline = iota
    41  	online
    42  	closed
    43  )
    44  
    45  // Hold the number of failed RPC calls due to networking errors
    46  var networkErrsCounter uint64
    47  
    48  // GetNetworkErrsCounter returns the number of failed RPC requests
    49  func GetNetworkErrsCounter() uint64 {
    50  	return atomic.LoadUint64(&networkErrsCounter)
    51  }
    52  
    53  // ResetNetworkErrsCounter resets the number of failed RPC requests
    54  func ResetNetworkErrsCounter() {
    55  	atomic.StoreUint64(&networkErrsCounter, 0)
    56  }
    57  
    58  // NetworkError - error type in case of errors related to http/transport
    59  // for ex. connection refused, connection reset, dns resolution failure etc.
    60  // All errors returned by storage-rest-server (ex errFileNotFound, errDiskNotFound) are not considered to be network errors.
    61  type NetworkError struct {
    62  	Err error
    63  }
    64  
    65  func (n *NetworkError) Error() string {
    66  	return n.Err.Error()
    67  }
    68  
    69  // Unwrap returns the error wrapped in NetworkError.
    70  func (n *NetworkError) Unwrap() error {
    71  	return n.Err
    72  }
    73  
    74  // Client - http based RPC client.
    75  type Client struct {
    76  	connected int32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
    77  
    78  	// HealthCheckFn is the function set to test for health.
    79  	// If not set the client will not keep track of health.
    80  	// Calling this returns true or false if the target
    81  	// is online or offline.
    82  	HealthCheckFn func() bool
    83  
    84  	// HealthCheckInterval will be the duration between re-connection attempts
    85  	// when a call has failed with a network error.
    86  	HealthCheckInterval time.Duration
    87  
    88  	// HealthCheckTimeout determines timeout for each call.
    89  	HealthCheckTimeout time.Duration
    90  
    91  	// MaxErrResponseSize is the maximum expected response size.
    92  	// Should only be modified before any calls are made.
    93  	MaxErrResponseSize int64
    94  
    95  	// ExpectTimeouts indicates if context timeouts are expected.
    96  	// This will not mark the client offline in these cases.
    97  	ExpectTimeouts bool
    98  
    99  	httpClient   *http.Client
   100  	url          *url.URL
   101  	newAuthToken func(audience string) string
   102  }
   103  
   104  // URL query separator constants
   105  const (
   106  	querySep = "?"
   107  )
   108  
   109  type restError string
   110  
   111  func (e restError) Error() string {
   112  	return string(e)
   113  }
   114  
   115  func (e restError) Timeout() bool {
   116  	return true
   117  }
   118  
   119  // Call - make a REST call with context.
   120  func (c *Client) Call(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) {
   121  	if !c.IsOnline() {
   122  		return nil, &NetworkError{Err: &url.Error{Op: method, URL: c.url.String(), Err: restError("remote server offline")}}
   123  	}
   124  	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url.String()+method+querySep+values.Encode(), body)
   125  	if err != nil {
   126  		return nil, &NetworkError{err}
   127  	}
   128  	req.Header.Set("Authorization", "Bearer "+c.newAuthToken(req.URL.RawQuery))
   129  	req.Header.Set("X-Minio-Time", time.Now().UTC().Format(time.RFC3339))
   130  	req.Header.Set("Expect", "100-continue")
   131  	if length > 0 {
   132  		req.ContentLength = length
   133  	}
   134  	resp, err := c.httpClient.Do(req)
   135  	if err != nil {
   136  		if c.HealthCheckFn != nil && xnet.IsNetworkOrHostDown(err, c.ExpectTimeouts) {
   137  			atomic.AddUint64(&networkErrsCounter, 1)
   138  			if c.MarkOffline() {
   139  				logger.LogIf(ctx, fmt.Errorf("Marking %s temporary offline; caused by %w", c.url.String(), err))
   140  			}
   141  		}
   142  		return nil, &NetworkError{err}
   143  	}
   144  
   145  	final := resp.Trailer.Get("FinalStatus")
   146  	if final != "" && final != "Success" {
   147  		defer xhttp.DrainBody(resp.Body)
   148  		return nil, errors.New(final)
   149  	}
   150  
   151  	if resp.StatusCode != http.StatusOK {
   152  		// If server returns 412 pre-condition failed, it would
   153  		// mean that authentication succeeded, but another
   154  		// side-channel check has failed, we shall take
   155  		// the client offline in such situations.
   156  		// generally all implementations should simply return
   157  		// 403, but in situations where there is a dependency
   158  		// with the caller to take the client offline purpose
   159  		// fully it should make sure to respond with '412'
   160  		// instead, see cmd/storage-rest-server.go for ideas.
   161  		if c.HealthCheckFn != nil && resp.StatusCode == http.StatusPreconditionFailed {
   162  			logger.LogIf(ctx, fmt.Errorf("Marking %s temporary offline; caused by PreconditionFailed with disk ID mismatch", c.url.String()))
   163  			c.MarkOffline()
   164  		}
   165  		defer xhttp.DrainBody(resp.Body)
   166  		// Limit the ReadAll(), just in case, because of a bug, the server responds with large data.
   167  		b, err := ioutil.ReadAll(io.LimitReader(resp.Body, c.MaxErrResponseSize))
   168  		if err != nil {
   169  			if c.HealthCheckFn != nil && xnet.IsNetworkOrHostDown(err, c.ExpectTimeouts) {
   170  				if c.MarkOffline() {
   171  					logger.LogIf(ctx, fmt.Errorf("Marking %s temporary offline; caused by %w", c.url.String(), err))
   172  				}
   173  			}
   174  			return nil, err
   175  		}
   176  		if len(b) > 0 {
   177  			return nil, errors.New(string(b))
   178  		}
   179  		return nil, errors.New(resp.Status)
   180  	}
   181  	return resp.Body, nil
   182  }
   183  
   184  // Close closes all idle connections of the underlying http client
   185  func (c *Client) Close() {
   186  	atomic.StoreInt32(&c.connected, closed)
   187  }
   188  
   189  // NewClient - returns new REST client.
   190  func NewClient(url *url.URL, tr http.RoundTripper, newAuthToken func(aud string) string) *Client {
   191  	// Transport is exactly same as Go default in https://golang.org/pkg/net/http/#RoundTripper
   192  	// except custom DialContext and TLSClientConfig.
   193  	return &Client{
   194  		httpClient:          &http.Client{Transport: tr},
   195  		url:                 url,
   196  		newAuthToken:        newAuthToken,
   197  		connected:           online,
   198  		MaxErrResponseSize:  4096,
   199  		HealthCheckInterval: 200 * time.Millisecond,
   200  		HealthCheckTimeout:  time.Second,
   201  	}
   202  }
   203  
   204  // IsOnline returns whether the client is likely to be online.
   205  func (c *Client) IsOnline() bool {
   206  	return atomic.LoadInt32(&c.connected) == online
   207  }
   208  
   209  // MarkOffline - will mark a client as being offline and spawns
   210  // a goroutine that will attempt to reconnect if HealthCheckFn is set.
   211  // returns true if the node changed state from online to offline
   212  func (c *Client) MarkOffline() bool {
   213  	// Start goroutine that will attempt to reconnect.
   214  	// If server is already trying to reconnect this will have no effect.
   215  	if c.HealthCheckFn != nil && atomic.CompareAndSwapInt32(&c.connected, online, offline) {
   216  		r := rand.New(rand.NewSource(time.Now().UnixNano()))
   217  		go func() {
   218  			for {
   219  				if atomic.LoadInt32(&c.connected) == closed {
   220  					return
   221  				}
   222  				if c.HealthCheckFn() {
   223  					if atomic.CompareAndSwapInt32(&c.connected, offline, online) {
   224  						logger.Info("Client %s online", c.url.String())
   225  					}
   226  					return
   227  				}
   228  				time.Sleep(time.Duration(r.Float64() * float64(c.HealthCheckInterval)))
   229  			}
   230  		}()
   231  		return true
   232  	}
   233  	return false
   234  }