storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/rest/client.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2018-2020 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package rest 18 19 import ( 20 "context" 21 "errors" 22 "fmt" 23 "io" 24 "io/ioutil" 25 "math/rand" 26 "net/http" 27 "net/url" 28 "sync/atomic" 29 "time" 30 31 xhttp "storj.io/minio/cmd/http" 32 "storj.io/minio/cmd/logger" 33 xnet "storj.io/minio/pkg/net" 34 ) 35 36 // DefaultTimeout - default REST timeout is 10 seconds. 37 const DefaultTimeout = 10 * time.Second 38 39 const ( 40 offline = iota 41 online 42 closed 43 ) 44 45 // Hold the number of failed RPC calls due to networking errors 46 var networkErrsCounter uint64 47 48 // GetNetworkErrsCounter returns the number of failed RPC requests 49 func GetNetworkErrsCounter() uint64 { 50 return atomic.LoadUint64(&networkErrsCounter) 51 } 52 53 // ResetNetworkErrsCounter resets the number of failed RPC requests 54 func ResetNetworkErrsCounter() { 55 atomic.StoreUint64(&networkErrsCounter, 0) 56 } 57 58 // NetworkError - error type in case of errors related to http/transport 59 // for ex. connection refused, connection reset, dns resolution failure etc. 60 // All errors returned by storage-rest-server (ex errFileNotFound, errDiskNotFound) are not considered to be network errors. 61 type NetworkError struct { 62 Err error 63 } 64 65 func (n *NetworkError) Error() string { 66 return n.Err.Error() 67 } 68 69 // Unwrap returns the error wrapped in NetworkError. 70 func (n *NetworkError) Unwrap() error { 71 return n.Err 72 } 73 74 // Client - http based RPC client. 75 type Client struct { 76 connected int32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG 77 78 // HealthCheckFn is the function set to test for health. 79 // If not set the client will not keep track of health. 80 // Calling this returns true or false if the target 81 // is online or offline. 82 HealthCheckFn func() bool 83 84 // HealthCheckInterval will be the duration between re-connection attempts 85 // when a call has failed with a network error. 86 HealthCheckInterval time.Duration 87 88 // HealthCheckTimeout determines timeout for each call. 89 HealthCheckTimeout time.Duration 90 91 // MaxErrResponseSize is the maximum expected response size. 92 // Should only be modified before any calls are made. 93 MaxErrResponseSize int64 94 95 // ExpectTimeouts indicates if context timeouts are expected. 96 // This will not mark the client offline in these cases. 97 ExpectTimeouts bool 98 99 httpClient *http.Client 100 url *url.URL 101 newAuthToken func(audience string) string 102 } 103 104 // URL query separator constants 105 const ( 106 querySep = "?" 107 ) 108 109 type restError string 110 111 func (e restError) Error() string { 112 return string(e) 113 } 114 115 func (e restError) Timeout() bool { 116 return true 117 } 118 119 // Call - make a REST call with context. 120 func (c *Client) Call(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) { 121 if !c.IsOnline() { 122 return nil, &NetworkError{Err: &url.Error{Op: method, URL: c.url.String(), Err: restError("remote server offline")}} 123 } 124 req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url.String()+method+querySep+values.Encode(), body) 125 if err != nil { 126 return nil, &NetworkError{err} 127 } 128 req.Header.Set("Authorization", "Bearer "+c.newAuthToken(req.URL.RawQuery)) 129 req.Header.Set("X-Minio-Time", time.Now().UTC().Format(time.RFC3339)) 130 req.Header.Set("Expect", "100-continue") 131 if length > 0 { 132 req.ContentLength = length 133 } 134 resp, err := c.httpClient.Do(req) 135 if err != nil { 136 if c.HealthCheckFn != nil && xnet.IsNetworkOrHostDown(err, c.ExpectTimeouts) { 137 atomic.AddUint64(&networkErrsCounter, 1) 138 if c.MarkOffline() { 139 logger.LogIf(ctx, fmt.Errorf("Marking %s temporary offline; caused by %w", c.url.String(), err)) 140 } 141 } 142 return nil, &NetworkError{err} 143 } 144 145 final := resp.Trailer.Get("FinalStatus") 146 if final != "" && final != "Success" { 147 defer xhttp.DrainBody(resp.Body) 148 return nil, errors.New(final) 149 } 150 151 if resp.StatusCode != http.StatusOK { 152 // If server returns 412 pre-condition failed, it would 153 // mean that authentication succeeded, but another 154 // side-channel check has failed, we shall take 155 // the client offline in such situations. 156 // generally all implementations should simply return 157 // 403, but in situations where there is a dependency 158 // with the caller to take the client offline purpose 159 // fully it should make sure to respond with '412' 160 // instead, see cmd/storage-rest-server.go for ideas. 161 if c.HealthCheckFn != nil && resp.StatusCode == http.StatusPreconditionFailed { 162 logger.LogIf(ctx, fmt.Errorf("Marking %s temporary offline; caused by PreconditionFailed with disk ID mismatch", c.url.String())) 163 c.MarkOffline() 164 } 165 defer xhttp.DrainBody(resp.Body) 166 // Limit the ReadAll(), just in case, because of a bug, the server responds with large data. 167 b, err := ioutil.ReadAll(io.LimitReader(resp.Body, c.MaxErrResponseSize)) 168 if err != nil { 169 if c.HealthCheckFn != nil && xnet.IsNetworkOrHostDown(err, c.ExpectTimeouts) { 170 if c.MarkOffline() { 171 logger.LogIf(ctx, fmt.Errorf("Marking %s temporary offline; caused by %w", c.url.String(), err)) 172 } 173 } 174 return nil, err 175 } 176 if len(b) > 0 { 177 return nil, errors.New(string(b)) 178 } 179 return nil, errors.New(resp.Status) 180 } 181 return resp.Body, nil 182 } 183 184 // Close closes all idle connections of the underlying http client 185 func (c *Client) Close() { 186 atomic.StoreInt32(&c.connected, closed) 187 } 188 189 // NewClient - returns new REST client. 190 func NewClient(url *url.URL, tr http.RoundTripper, newAuthToken func(aud string) string) *Client { 191 // Transport is exactly same as Go default in https://golang.org/pkg/net/http/#RoundTripper 192 // except custom DialContext and TLSClientConfig. 193 return &Client{ 194 httpClient: &http.Client{Transport: tr}, 195 url: url, 196 newAuthToken: newAuthToken, 197 connected: online, 198 MaxErrResponseSize: 4096, 199 HealthCheckInterval: 200 * time.Millisecond, 200 HealthCheckTimeout: time.Second, 201 } 202 } 203 204 // IsOnline returns whether the client is likely to be online. 205 func (c *Client) IsOnline() bool { 206 return atomic.LoadInt32(&c.connected) == online 207 } 208 209 // MarkOffline - will mark a client as being offline and spawns 210 // a goroutine that will attempt to reconnect if HealthCheckFn is set. 211 // returns true if the node changed state from online to offline 212 func (c *Client) MarkOffline() bool { 213 // Start goroutine that will attempt to reconnect. 214 // If server is already trying to reconnect this will have no effect. 215 if c.HealthCheckFn != nil && atomic.CompareAndSwapInt32(&c.connected, online, offline) { 216 r := rand.New(rand.NewSource(time.Now().UnixNano())) 217 go func() { 218 for { 219 if atomic.LoadInt32(&c.connected) == closed { 220 return 221 } 222 if c.HealthCheckFn() { 223 if atomic.CompareAndSwapInt32(&c.connected, offline, online) { 224 logger.Info("Client %s online", c.url.String()) 225 } 226 return 227 } 228 time.Sleep(time.Duration(r.Float64() * float64(c.HealthCheckInterval))) 229 } 230 }() 231 return true 232 } 233 return false 234 }