github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/cmn/backend.go (about)

     1  // Package cmn provides common constants, types, and utilities for AIS clients
     2  // and AIStore.
     3  /*
     4   * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
     5   */
     6  package cmn
     7  
     8  import (
     9  	"encoding/base64"
    10  	"encoding/hex"
    11  	"strconv"
    12  	"strings"
    13  
    14  	"github.com/NVIDIA/aistore/cmn/debug"
    15  )
    16  
    17  type backendFuncs struct {
    18  	EncodeVersion func(v any) (version string, isSet bool)
    19  	EncodeCksum   func(v any) (cksumValue string, isSet bool)
    20  }
    21  
    22  // from https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html
    23  // "The ETag may or may not be an MD5 digest of the object data. Whether or
    24  // not it is depends on how the object was created and how it is encrypted..."
    25  const AwsMultipartDelim = "-"
    26  
    27  func IsS3MultipartEtag(etag string) bool {
    28  	return strings.Contains(etag, AwsMultipartDelim)
    29  }
    30  
    31  func awsIsVersionSet(version *string) bool {
    32  	return version != nil && *version != "" && *version != "null"
    33  }
    34  
    35  // unquote checksum, ETag, and version
    36  // e.g., https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
    37  func UnquoteCEV(val string) string { return strings.Trim(val, "\"") }
    38  
    39  var BackendHelpers = struct {
    40  	Amazon backendFuncs
    41  	Azure  backendFuncs
    42  	Google backendFuncs
    43  	HTTP   backendFuncs
    44  }{
    45  	Amazon: backendFuncs{
    46  		EncodeVersion: func(v any) (string, bool) {
    47  			switch x := v.(type) {
    48  			case *string:
    49  				if awsIsVersionSet(x) {
    50  					return *x, true
    51  				}
    52  				return "", false
    53  			case string:
    54  				if awsIsVersionSet(&x) {
    55  					return x, true
    56  				}
    57  				return x, false
    58  			default:
    59  				debug.FailTypeCast(v)
    60  				return "", false
    61  			}
    62  		},
    63  		EncodeCksum: func(v any) (string, bool) {
    64  			switch x := v.(type) {
    65  			case *string:
    66  				if IsS3MultipartEtag(*x) {
    67  					return *x, true // return as-is multipart
    68  				}
    69  				return UnquoteCEV(*x), true
    70  			case string:
    71  				return x, true
    72  			default:
    73  				debug.FailTypeCast(v)
    74  				return "", false
    75  			}
    76  		},
    77  	},
    78  	Google: backendFuncs{
    79  		EncodeVersion: func(v any) (string, bool) {
    80  			switch x := v.(type) {
    81  			case string:
    82  				return x, x != ""
    83  			case int64:
    84  				return strconv.FormatInt(x, 10), true
    85  			default:
    86  				debug.FailTypeCast(v)
    87  				return "", false
    88  			}
    89  		},
    90  		EncodeCksum: func(v any) (string, bool) {
    91  			switch x := v.(type) {
    92  			case string:
    93  				decoded, err := base64.StdEncoding.DecodeString(x)
    94  				if err != nil {
    95  					return "", false
    96  				}
    97  				return hex.EncodeToString(decoded), true
    98  			case []byte:
    99  				return hex.EncodeToString(x), true
   100  			case uint32:
   101  				// Encode a uint32 as Base64 in big-endian byte order.
   102  				// See: https://cloud.google.com/storage/docs/xml-api/reference-headers#xgooghash.
   103  				b := []byte{byte(x >> 24), byte(x >> 16), byte(x >> 8), byte(x)}
   104  				return base64.StdEncoding.EncodeToString(b), true
   105  			default:
   106  				debug.FailTypeCast(v)
   107  				return "", false
   108  			}
   109  		},
   110  	},
   111  	HTTP: backendFuncs{
   112  		EncodeVersion: func(v any) (string, bool) {
   113  			switch x := v.(type) {
   114  			case string:
   115  				// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag
   116  				x = strings.TrimPrefix(x, "W/")
   117  				x = UnquoteCEV(x)
   118  				return x, x != ""
   119  			default:
   120  				debug.FailTypeCast(v)
   121  				return "", false
   122  			}
   123  		},
   124  	},
   125  }