github.com/matrixorigin/matrixone@v1.2.0/pkg/fileservice/object_storage_arguments.go (about)

     1  // Copyright 2023 Matrix Origin
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package fileservice
    16  
    17  import (
    18  	"encoding/json"
    19  	"net/http"
    20  	"net/url"
    21  	"strconv"
    22  	"strings"
    23  
    24  	"github.com/matrixorigin/matrixone/pkg/common/moerr"
    25  )
    26  
    27  type ObjectStorageArguments struct {
    28  	// misc
    29  	Name                 string `toml:"name"`
    30  	KeyPrefix            string `toml:"key-prefix"`
    31  	SharedConfigProfile  string `toml:"shared-config-profile"`
    32  	NoDefaultCredentials bool   `toml:"no-default-credentials"`
    33  	NoBucketValidation   bool   `toml:"no-bucket-validation"`
    34  
    35  	// s3
    36  	Bucket    string   `toml:"bucket"`
    37  	Endpoint  string   `toml:"endpoint"`
    38  	IsMinio   bool     `toml:"is-minio"`
    39  	Region    string   `toml:"region"`
    40  	CertFiles []string `toml:"cert-files"`
    41  
    42  	// credentials
    43  	RoleARN         string `json:"-" toml:"role-arn"`
    44  	BearerToken     string `json:"-" toml:"bearer-token"`
    45  	ExternalID      string `json:"-" toml:"external-id"`
    46  	KeyID           string `json:"-" toml:"key-id"`
    47  	KeySecret       string `json:"-" toml:"key-secret"`
    48  	RAMRole         string `json:"-" toml:"ram-role"`
    49  	RoleSessionName string `json:"-" toml:"role-session-name"`
    50  	SecurityToken   string `json:"-" toml:"security-token"`
    51  	SessionToken    string `json:"-" toml:"session-token"`
    52  }
    53  
    54  func (o ObjectStorageArguments) String() string {
    55  	bs, err := json.Marshal(o)
    56  	if err != nil {
    57  		panic(err)
    58  	}
    59  	return string(bs)
    60  }
    61  
    62  func (o *ObjectStorageArguments) SetFromString(arguments []string) error {
    63  	for _, pair := range arguments {
    64  		key, value, ok := strings.Cut(pair, "=")
    65  		if !ok {
    66  			return moerr.NewInvalidInputNoCtx("invalid S3 argument: %s", pair)
    67  		}
    68  
    69  		switch strings.ToLower(key) {
    70  
    71  		case "name":
    72  			o.Name = value
    73  		case "prefix", "key-prefix":
    74  			o.KeyPrefix = value
    75  		case "shared-config-profile":
    76  			o.SharedConfigProfile = value
    77  		case "no-bucket-validation":
    78  			b, err := strconv.ParseBool(value)
    79  			if err == nil {
    80  				o.NoBucketValidation = b
    81  			}
    82  		case "no-default-credentials":
    83  			b, err := strconv.ParseBool(value)
    84  			if err == nil {
    85  				o.NoDefaultCredentials = b
    86  			}
    87  
    88  		case "bucket":
    89  			o.Bucket = value
    90  		case "endpoint":
    91  			o.Endpoint = value
    92  		case "is-minio", "minio":
    93  			o.IsMinio = value != "false" && value != "0"
    94  		case "region":
    95  			o.Region = value
    96  		case "cert-files":
    97  			o.CertFiles = strings.Split(value, ",")
    98  
    99  		case "role-arn":
   100  			o.RoleARN = value
   101  		case "bearer-token":
   102  			o.BearerToken = value
   103  		case "external-id":
   104  			o.ExternalID = value
   105  		case "key", "key-id":
   106  			o.KeyID = value
   107  		case "secret", "key-secret", "secret-id":
   108  			o.KeySecret = value
   109  		case "ram-role":
   110  			o.RAMRole = value
   111  		case "role-session-name":
   112  			o.RoleSessionName = value
   113  		case "security-token":
   114  			o.SecurityToken = value
   115  		case "token", "session-token":
   116  			o.SessionToken = value
   117  
   118  		default:
   119  			return moerr.NewInvalidInputNoCtx("invalid S3 argument: %s", pair)
   120  		}
   121  
   122  	}
   123  	return nil
   124  }
   125  
   126  func (o *ObjectStorageArguments) validate() error {
   127  
   128  	// validate endpoint
   129  	var endpointURL *url.URL
   130  	if o.Endpoint != "" {
   131  		var err error
   132  		endpointURL, err = url.Parse(o.Endpoint)
   133  		if err != nil {
   134  			return err
   135  		}
   136  		if endpointURL.Scheme == "" {
   137  			endpointURL.Scheme = "https"
   138  		}
   139  		o.Endpoint = endpointURL.String()
   140  	}
   141  
   142  	// region
   143  	if o.Region == "" {
   144  		// try to get region from bucket
   145  		// only works for AWS S3
   146  		resp, err := http.Head("https://" + o.Bucket + ".s3.amazonaws.com")
   147  		if err == nil {
   148  			if value := resp.Header.Get("x-amz-bucket-region"); value != "" {
   149  				o.Region = value
   150  			}
   151  		}
   152  	}
   153  
   154  	// role session name
   155  	if o.RoleSessionName == "" {
   156  		o.RoleSessionName = "mo-service"
   157  	}
   158  
   159  	return nil
   160  }
   161  
   162  func (o *ObjectStorageArguments) shouldLoadDefaultCredentials() bool {
   163  
   164  	// default credentials enabled
   165  	if !o.NoDefaultCredentials {
   166  		return true
   167  	}
   168  
   169  	// default credentials disabled, but role arn is not empty
   170  	if o.RoleARN != "" {
   171  		return true
   172  	}
   173  
   174  	return false
   175  }