github.com/minio/minio-go/v6@v6.0.57/utils.go (about) 1 /* 2 * MinIO Go Library for Amazon S3 Compatible Cloud Storage 3 * Copyright 2015-2017 MinIO, Inc. 4 * 5 * Licensed under the Apache License, Version 2.0 (the "License"); 6 * you may not use this file except in compliance with the License. 7 * You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18 package minio 19 20 import ( 21 "crypto/md5" 22 "encoding/base64" 23 "encoding/hex" 24 "encoding/xml" 25 "hash" 26 "io" 27 "io/ioutil" 28 "net" 29 "net/http" 30 "net/url" 31 "regexp" 32 "strconv" 33 "strings" 34 "sync" 35 "time" 36 37 md5simd "github.com/minio/md5-simd" 38 "github.com/minio/minio-go/v6/pkg/s3utils" 39 "github.com/minio/sha256-simd" 40 ) 41 42 func trimEtag(etag string) string { 43 etag = strings.TrimPrefix(etag, "\"") 44 return strings.TrimSuffix(etag, "\"") 45 } 46 47 // xmlDecoder provide decoded value in xml. 48 func xmlDecoder(body io.Reader, v interface{}) error { 49 d := xml.NewDecoder(body) 50 return d.Decode(v) 51 } 52 53 // sum256 calculate sha256sum for an input byte array, returns hex encoded. 54 func sum256Hex(data []byte) string { 55 hash := newSHA256Hasher() 56 defer hash.Close() 57 hash.Write(data) 58 return hex.EncodeToString(hash.Sum(nil)) 59 } 60 61 // sumMD5Base64 calculate md5sum for an input byte array, returns base64 encoded. 62 func sumMD5Base64(data []byte) string { 63 hash := newMd5Hasher() 64 defer hash.Close() 65 hash.Write(data) 66 return base64.StdEncoding.EncodeToString(hash.Sum(nil)) 67 } 68 69 // getEndpointURL - construct a new endpoint. 70 func getEndpointURL(endpoint string, secure bool) (*url.URL, error) { 71 if strings.Contains(endpoint, ":") { 72 host, _, err := net.SplitHostPort(endpoint) 73 if err != nil { 74 return nil, err 75 } 76 if !s3utils.IsValidIP(host) && !s3utils.IsValidDomain(host) { 77 msg := "Endpoint: " + endpoint + " does not follow ip address or domain name standards." 78 return nil, ErrInvalidArgument(msg) 79 } 80 } else { 81 if !s3utils.IsValidIP(endpoint) && !s3utils.IsValidDomain(endpoint) { 82 msg := "Endpoint: " + endpoint + " does not follow ip address or domain name standards." 83 return nil, ErrInvalidArgument(msg) 84 } 85 } 86 // If secure is false, use 'http' scheme. 87 scheme := "https" 88 if !secure { 89 scheme = "http" 90 } 91 92 // Construct a secured endpoint URL. 93 endpointURLStr := scheme + "://" + endpoint 94 endpointURL, err := url.Parse(endpointURLStr) 95 if err != nil { 96 return nil, err 97 } 98 99 // Validate incoming endpoint URL. 100 if err := isValidEndpointURL(*endpointURL); err != nil { 101 return nil, err 102 } 103 return endpointURL, nil 104 } 105 106 // closeResponse close non nil response with any response Body. 107 // convenient wrapper to drain any remaining data on response body. 108 // 109 // Subsequently this allows golang http RoundTripper 110 // to re-use the same connection for future requests. 111 func closeResponse(resp *http.Response) { 112 // Callers should close resp.Body when done reading from it. 113 // If resp.Body is not closed, the Client's underlying RoundTripper 114 // (typically Transport) may not be able to re-use a persistent TCP 115 // connection to the server for a subsequent "keep-alive" request. 116 if resp != nil && resp.Body != nil { 117 // Drain any remaining Body and then close the connection. 118 // Without this closing connection would disallow re-using 119 // the same connection for future uses. 120 // - http://stackoverflow.com/a/17961593/4465767 121 io.Copy(ioutil.Discard, resp.Body) 122 resp.Body.Close() 123 } 124 } 125 126 var ( 127 // Hex encoded string of nil sha256sum bytes. 128 emptySHA256Hex = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" 129 130 // Sentinel URL is the default url value which is invalid. 131 sentinelURL = url.URL{} 132 ) 133 134 // Verify if input endpoint URL is valid. 135 func isValidEndpointURL(endpointURL url.URL) error { 136 if endpointURL == sentinelURL { 137 return ErrInvalidArgument("Endpoint url cannot be empty.") 138 } 139 if endpointURL.Path != "/" && endpointURL.Path != "" { 140 return ErrInvalidArgument("Endpoint url cannot have fully qualified paths.") 141 } 142 if strings.Contains(endpointURL.Host, ".s3.amazonaws.com") { 143 if !s3utils.IsAmazonEndpoint(endpointURL) { 144 return ErrInvalidArgument("Amazon S3 endpoint should be 's3.amazonaws.com'.") 145 } 146 } 147 if strings.Contains(endpointURL.Host, ".googleapis.com") { 148 if !s3utils.IsGoogleEndpoint(endpointURL) { 149 return ErrInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'.") 150 } 151 } 152 return nil 153 } 154 155 // Verify if input expires value is valid. 156 func isValidExpiry(expires time.Duration) error { 157 expireSeconds := int64(expires / time.Second) 158 if expireSeconds < 1 { 159 return ErrInvalidArgument("Expires cannot be lesser than 1 second.") 160 } 161 if expireSeconds > 604800 { 162 return ErrInvalidArgument("Expires cannot be greater than 7 days.") 163 } 164 return nil 165 } 166 167 // Extract only necessary metadata header key/values by 168 // filtering them out with a list of custom header keys. 169 func extractObjMetadata(header http.Header) http.Header { 170 preserveKeys := []string{ 171 "Content-Type", 172 "Cache-Control", 173 "Content-Encoding", 174 "Content-Language", 175 "Content-Disposition", 176 "X-Amz-Storage-Class", 177 "X-Amz-Object-Lock-Mode", 178 "X-Amz-Object-Lock-Retain-Until-Date", 179 "X-Amz-Object-Lock-Legal-Hold", 180 "X-Amz-Website-Redirect-Location", 181 "X-Amz-Server-Side-Encryption", 182 "X-Amz-Tagging-Count", 183 "X-Amz-Meta-", 184 // Add new headers to be preserved. 185 // if you add new headers here, please extend 186 // PutObjectOptions{} to preserve them 187 // upon upload as well. 188 } 189 filteredHeader := make(http.Header) 190 for k, v := range header { 191 var found bool 192 for _, prefix := range preserveKeys { 193 if !strings.HasPrefix(k, prefix) { 194 continue 195 } 196 found = true 197 break 198 } 199 if found { 200 filteredHeader[k] = v 201 } 202 } 203 return filteredHeader 204 } 205 206 // ToObjectInfo converts http header values into ObjectInfo type, 207 // extracts metadata and fills in all the necessary fields in ObjectInfo. 208 func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectInfo, error) { 209 var err error 210 // Trim off the odd double quotes from ETag in the beginning and end. 211 etag := trimEtag(h.Get("ETag")) 212 213 // Parse content length is exists 214 var size int64 = -1 215 contentLengthStr := h.Get("Content-Length") 216 if contentLengthStr != "" { 217 size, err = strconv.ParseInt(contentLengthStr, 10, 64) 218 if err != nil { 219 // Content-Length is not valid 220 return ObjectInfo{}, ErrorResponse{ 221 Code: "InternalError", 222 Message: "Content-Length is invalid. " + reportIssue, 223 BucketName: bucketName, 224 Key: objectName, 225 RequestID: h.Get("x-amz-request-id"), 226 HostID: h.Get("x-amz-id-2"), 227 Region: h.Get("x-amz-bucket-region"), 228 } 229 } 230 } 231 232 // Parse Last-Modified has http time format. 233 date, err := time.Parse(http.TimeFormat, h.Get("Last-Modified")) 234 if err != nil { 235 return ObjectInfo{}, ErrorResponse{ 236 Code: "InternalError", 237 Message: "Last-Modified time format is invalid. " + reportIssue, 238 BucketName: bucketName, 239 Key: objectName, 240 RequestID: h.Get("x-amz-request-id"), 241 HostID: h.Get("x-amz-id-2"), 242 Region: h.Get("x-amz-bucket-region"), 243 } 244 } 245 246 // Fetch content type if any present. 247 contentType := strings.TrimSpace(h.Get("Content-Type")) 248 if contentType == "" { 249 contentType = "application/octet-stream" 250 } 251 252 expiryStr := h.Get("Expires") 253 var expTime time.Time 254 if t, err := time.Parse(http.TimeFormat, expiryStr); err == nil { 255 expTime = t.UTC() 256 } 257 258 metadata := extractObjMetadata(h) 259 userMetadata := make(map[string]string) 260 for k, v := range metadata { 261 if strings.HasPrefix(k, "X-Amz-Meta-") { 262 userMetadata[strings.TrimPrefix(k, "X-Amz-Meta-")] = v[0] 263 } 264 } 265 userTags := s3utils.TagDecode(h.Get(amzTaggingHeader)) 266 267 // Save object metadata info. 268 return ObjectInfo{ 269 ETag: etag, 270 Key: objectName, 271 Size: size, 272 LastModified: date, 273 ContentType: contentType, 274 Expires: expTime, 275 // Extract only the relevant header keys describing the object. 276 // following function filters out a list of standard set of keys 277 // which are not part of object metadata. 278 Metadata: metadata, 279 UserMetadata: userMetadata, 280 UserTags: userTags, 281 }, nil 282 } 283 284 // regCred matches credential string in HTTP header 285 var regCred = regexp.MustCompile("Credential=([A-Z0-9]+)/") 286 287 // regCred matches signature string in HTTP header 288 var regSign = regexp.MustCompile("Signature=([[0-9a-f]+)") 289 290 // Redact out signature value from authorization string. 291 func redactSignature(origAuth string) string { 292 if !strings.HasPrefix(origAuth, signV4Algorithm) { 293 // Set a temporary redacted auth 294 return "AWS **REDACTED**:**REDACTED**" 295 } 296 297 /// Signature V4 authorization header. 298 299 // Strip out accessKeyID from: 300 // Credential=<access-key-id>/<date>/<aws-region>/<aws-service>/aws4_request 301 newAuth := regCred.ReplaceAllString(origAuth, "Credential=**REDACTED**/") 302 303 // Strip out 256-bit signature from: Signature=<256-bit signature> 304 return regSign.ReplaceAllString(newAuth, "Signature=**REDACTED**") 305 } 306 307 // Get default location returns the location based on the input 308 // URL `u`, if region override is provided then all location 309 // defaults to regionOverride. 310 // 311 // If no other cases match then the location is set to `us-east-1` 312 // as a last resort. 313 func getDefaultLocation(u url.URL, regionOverride string) (location string) { 314 if regionOverride != "" { 315 return regionOverride 316 } 317 region := s3utils.GetRegionFromURL(u) 318 if region == "" { 319 region = "us-east-1" 320 } 321 return region 322 } 323 324 var supportedHeaders = []string{ 325 "content-type", 326 "cache-control", 327 "content-encoding", 328 "content-disposition", 329 "content-language", 330 "x-amz-website-redirect-location", 331 "x-amz-object-lock-mode", 332 "x-amz-metadata-directive", 333 "x-amz-object-lock-retain-until-date", 334 "expires", 335 // Add more supported headers here. 336 } 337 338 // isStorageClassHeader returns true if the header is a supported storage class header 339 func isStorageClassHeader(headerKey string) bool { 340 return strings.EqualFold(amzStorageClass, headerKey) 341 } 342 343 // isStandardHeader returns true if header is a supported header and not a custom header 344 func isStandardHeader(headerKey string) bool { 345 key := strings.ToLower(headerKey) 346 for _, header := range supportedHeaders { 347 if strings.ToLower(header) == key { 348 return true 349 } 350 } 351 return false 352 } 353 354 // sseHeaders is list of server side encryption headers 355 var sseHeaders = []string{ 356 "x-amz-server-side-encryption", 357 "x-amz-server-side-encryption-aws-kms-key-id", 358 "x-amz-server-side-encryption-context", 359 "x-amz-server-side-encryption-customer-algorithm", 360 "x-amz-server-side-encryption-customer-key", 361 "x-amz-server-side-encryption-customer-key-MD5", 362 } 363 364 // isSSEHeader returns true if header is a server side encryption header. 365 func isSSEHeader(headerKey string) bool { 366 key := strings.ToLower(headerKey) 367 for _, h := range sseHeaders { 368 if strings.ToLower(h) == key { 369 return true 370 } 371 } 372 return false 373 } 374 375 // isAmzHeader returns true if header is a x-amz-meta-* or x-amz-acl header. 376 func isAmzHeader(headerKey string) bool { 377 key := strings.ToLower(headerKey) 378 379 return strings.HasPrefix(key, "x-amz-meta-") || strings.HasPrefix(key, "x-amz-grant-") || key == "x-amz-acl" || isSSEHeader(headerKey) 380 } 381 382 var md5Pool = sync.Pool{New: func() interface{} { return md5.New() }} 383 var sha256Pool = sync.Pool{New: func() interface{} { return sha256.New() }} 384 385 func newMd5Hasher() md5simd.Hasher { 386 return hashWrapper{Hash: md5Pool.New().(hash.Hash), isMD5: true} 387 } 388 389 func newSHA256Hasher() md5simd.Hasher { 390 return hashWrapper{Hash: sha256Pool.New().(hash.Hash), isSHA256: true} 391 } 392 393 // hashWrapper implements the md5simd.Hasher interface. 394 type hashWrapper struct { 395 hash.Hash 396 isMD5 bool 397 isSHA256 bool 398 } 399 400 // Close will put the hasher back into the pool. 401 func (m hashWrapper) Close() { 402 if m.isMD5 && m.Hash != nil { 403 m.Reset() 404 md5Pool.Put(m.Hash) 405 } 406 if m.isSHA256 && m.Hash != nil { 407 m.Reset() 408 sha256Pool.Put(m.Hash) 409 } 410 m.Hash = nil 411 }