github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/oracleobjectstorage/options.go (about)

     1  //go:build !plan9 && !solaris && !js
     2  
     3  package oracleobjectstorage
     4  
     5  import (
     6  	"time"
     7  
     8  	"github.com/rclone/rclone/fs"
     9  	"github.com/rclone/rclone/fs/config"
    10  	"github.com/rclone/rclone/lib/encoder"
    11  )
    12  
    13  const (
    14  	maxSizeForCopy             = 4768 * 1024 * 1024
    15  	maxUploadParts             = 10000
    16  	defaultUploadConcurrency   = 10
    17  	minChunkSize               = fs.SizeSuffix(5 * 1024 * 1024)
    18  	defaultUploadCutoff        = fs.SizeSuffix(200 * 1024 * 1024)
    19  	maxUploadCutoff            = fs.SizeSuffix(5 * 1024 * 1024 * 1024)
    20  	minSleep                   = 10 * time.Millisecond
    21  	defaultCopyTimeoutDuration = fs.Duration(time.Minute)
    22  )
    23  
    24  const (
    25  	userPrincipal     = "user_principal_auth"
    26  	instancePrincipal = "instance_principal_auth"
    27  	resourcePrincipal = "resource_principal_auth"
    28  	workloadIdentity  = "workload_identity_auth"
    29  	environmentAuth   = "env_auth"
    30  	noAuth            = "no_auth"
    31  
    32  	userPrincipalHelpText = `use an OCI user and an API key for authentication.
    33  you’ll need to put in a config file your tenancy OCID, user OCID, region, the path, fingerprint to an API key.
    34  https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm`
    35  
    36  	instancePrincipalHelpText = `use instance principals to authorize an instance to make API calls. 
    37  each instance has its own identity, and authenticates using the certificates that are read from instance metadata. 
    38  https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfrominstances.htm`
    39  
    40  	workloadIdentityHelpText = `use workload identity to grant OCI Container Engine for Kubernetes workloads policy-driven access to OCI resources using OCI Identity and Access Management (IAM).
    41  https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contenggrantingworkloadaccesstoresources.htm`
    42  	resourcePrincipalHelpText = `use resource principals to make API calls`
    43  
    44  	environmentAuthHelpText = `automatically pickup the credentials from runtime(env), first one to provide auth wins`
    45  
    46  	noAuthHelpText = `no credentials needed, this is typically for reading public buckets`
    47  )
    48  
    49  // Options defines the configuration for this backend
    50  type Options struct {
    51  	Provider             string               `config:"provider"`
    52  	Compartment          string               `config:"compartment"`
    53  	Namespace            string               `config:"namespace"`
    54  	Region               string               `config:"region"`
    55  	Endpoint             string               `config:"endpoint"`
    56  	Enc                  encoder.MultiEncoder `config:"encoding"`
    57  	ConfigFile           string               `config:"config_file"`
    58  	ConfigProfile        string               `config:"config_profile"`
    59  	UploadCutoff         fs.SizeSuffix        `config:"upload_cutoff"`
    60  	ChunkSize            fs.SizeSuffix        `config:"chunk_size"`
    61  	MaxUploadParts       int                  `config:"max_upload_parts"`
    62  	UploadConcurrency    int                  `config:"upload_concurrency"`
    63  	DisableChecksum      bool                 `config:"disable_checksum"`
    64  	CopyCutoff           fs.SizeSuffix        `config:"copy_cutoff"`
    65  	CopyTimeout          fs.Duration          `config:"copy_timeout"`
    66  	StorageTier          string               `config:"storage_tier"`
    67  	LeavePartsOnError    bool                 `config:"leave_parts_on_error"`
    68  	AttemptResumeUpload  bool                 `config:"attempt_resume_upload"`
    69  	NoCheckBucket        bool                 `config:"no_check_bucket"`
    70  	SSEKMSKeyID          string               `config:"sse_kms_key_id"`
    71  	SSECustomerAlgorithm string               `config:"sse_customer_algorithm"`
    72  	SSECustomerKey       string               `config:"sse_customer_key"`
    73  	SSECustomerKeyFile   string               `config:"sse_customer_key_file"`
    74  	SSECustomerKeySha256 string               `config:"sse_customer_key_sha256"`
    75  }
    76  
    77  func newOptions() []fs.Option {
    78  	return []fs.Option{{
    79  		Name:     fs.ConfigProvider,
    80  		Help:     "Choose your Auth Provider",
    81  		Required: true,
    82  		Default:  environmentAuth,
    83  		Examples: []fs.OptionExample{{
    84  			Value: environmentAuth,
    85  			Help:  environmentAuthHelpText,
    86  		}, {
    87  			Value: userPrincipal,
    88  			Help:  userPrincipalHelpText,
    89  		}, {
    90  			Value: instancePrincipal,
    91  			Help:  instancePrincipalHelpText,
    92  		}, {
    93  			Value: workloadIdentity,
    94  			Help:  workloadIdentityHelpText,
    95  		}, {
    96  			Value: resourcePrincipal,
    97  			Help:  resourcePrincipalHelpText,
    98  		}, {
    99  			Value: noAuth,
   100  			Help:  noAuthHelpText,
   101  		}},
   102  	}, {
   103  		Name:      "namespace",
   104  		Help:      "Object storage namespace",
   105  		Required:  true,
   106  		Sensitive: true,
   107  	}, {
   108  		Name:      "compartment",
   109  		Help:      "Object storage compartment OCID",
   110  		Provider:  "!no_auth",
   111  		Required:  true,
   112  		Sensitive: true,
   113  	}, {
   114  		Name:     "region",
   115  		Help:     "Object storage Region",
   116  		Required: true,
   117  	}, {
   118  		Name:     "endpoint",
   119  		Help:     "Endpoint for Object storage API.\n\nLeave blank to use the default endpoint for the region.",
   120  		Required: false,
   121  	}, {
   122  		Name:     "config_file",
   123  		Help:     "Path to OCI config file",
   124  		Provider: userPrincipal,
   125  		Default:  "~/.oci/config",
   126  		Examples: []fs.OptionExample{{
   127  			Value: "~/.oci/config",
   128  			Help:  "oci configuration file location",
   129  		}},
   130  	}, {
   131  		Name:     "config_profile",
   132  		Help:     "Profile name inside the oci config file",
   133  		Provider: userPrincipal,
   134  		Default:  "Default",
   135  		Examples: []fs.OptionExample{{
   136  			Value: "Default",
   137  			Help:  "Use the default profile",
   138  		}},
   139  	}, {
   140  		// Mapping from here: https://github.com/oracle/oci-go-sdk/blob/master/objectstorage/storage_tier.go
   141  		Name:     "storage_tier",
   142  		Help:     "The storage class to use when storing new objects in storage. https://docs.oracle.com/en-us/iaas/Content/Object/Concepts/understandingstoragetiers.htm",
   143  		Default:  "Standard",
   144  		Advanced: true,
   145  		Examples: []fs.OptionExample{{
   146  			Value: "Standard",
   147  			Help:  "Standard storage tier, this is the default tier",
   148  		}, {
   149  			Value: "InfrequentAccess",
   150  			Help:  "InfrequentAccess storage tier",
   151  		}, {
   152  			Value: "Archive",
   153  			Help:  "Archive storage tier",
   154  		}},
   155  	}, {
   156  		Name: "upload_cutoff",
   157  		Help: `Cutoff for switching to chunked upload.
   158  
   159  Any files larger than this will be uploaded in chunks of chunk_size.
   160  The minimum is 0 and the maximum is 5 GiB.`,
   161  		Default:  defaultUploadCutoff,
   162  		Advanced: true,
   163  	}, {
   164  		Name: "chunk_size",
   165  		Help: `Chunk size to use for uploading.
   166  
   167  When uploading files larger than upload_cutoff or files with unknown
   168  size (e.g. from "rclone rcat" or uploaded with "rclone mount" they will be uploaded 
   169  as multipart uploads using this chunk size.
   170  
   171  Note that "upload_concurrency" chunks of this size are buffered
   172  in memory per transfer.
   173  
   174  If you are transferring large files over high-speed links and you have
   175  enough memory, then increasing this will speed up the transfers.
   176  
   177  Rclone will automatically increase the chunk size when uploading a
   178  large file of known size to stay below the 10,000 chunks limit.
   179  
   180  Files of unknown size are uploaded with the configured
   181  chunk_size. Since the default chunk size is 5 MiB and there can be at
   182  most 10,000 chunks, this means that by default the maximum size of
   183  a file you can stream upload is 48 GiB.  If you wish to stream upload
   184  larger files then you will need to increase chunk_size.
   185  
   186  Increasing the chunk size decreases the accuracy of the progress
   187  statistics displayed with "-P" flag.
   188  `,
   189  		Default:  minChunkSize,
   190  		Advanced: true,
   191  	}, {
   192  		Name: "max_upload_parts",
   193  		Help: `Maximum number of parts in a multipart upload.
   194  
   195  This option defines the maximum number of multipart chunks to use
   196  when doing a multipart upload.
   197  
   198  OCI has max parts limit of 10,000 chunks.
   199  
   200  Rclone will automatically increase the chunk size when uploading a
   201  large file of a known size to stay below this number of chunks limit.
   202  `,
   203  		Default:  maxUploadParts,
   204  		Advanced: true,
   205  	}, {
   206  		Name: "upload_concurrency",
   207  		Help: `Concurrency for multipart uploads.
   208  
   209  This is the number of chunks of the same file that are uploaded
   210  concurrently.
   211  
   212  If you are uploading small numbers of large files over high-speed links
   213  and these uploads do not fully utilize your bandwidth, then increasing
   214  this may help to speed up the transfers.`,
   215  		Default:  defaultUploadConcurrency,
   216  		Advanced: true,
   217  	}, {
   218  		Name: "copy_cutoff",
   219  		Help: `Cutoff for switching to multipart copy.
   220  
   221  Any files larger than this that need to be server-side copied will be
   222  copied in chunks of this size.
   223  
   224  The minimum is 0 and the maximum is 5 GiB.`,
   225  		Default:  fs.SizeSuffix(maxSizeForCopy),
   226  		Advanced: true,
   227  	}, {
   228  		Name: "copy_timeout",
   229  		Help: `Timeout for copy.
   230  
   231  Copy is an asynchronous operation, specify timeout to wait for copy to succeed
   232  `,
   233  		Default:  defaultCopyTimeoutDuration,
   234  		Advanced: true,
   235  	}, {
   236  		Name: "disable_checksum",
   237  		Help: `Don't store MD5 checksum with object metadata.
   238  
   239  Normally rclone will calculate the MD5 checksum of the input before
   240  uploading it so it can add it to metadata on the object. This is great
   241  for data integrity checking but can cause long delays for large files
   242  to start uploading.`,
   243  		Default:  false,
   244  		Advanced: true,
   245  	}, {
   246  		Name:     config.ConfigEncoding,
   247  		Help:     config.ConfigEncodingHelp,
   248  		Advanced: true,
   249  		// Any UTF-8 character is valid in a key, however it can't handle
   250  		// invalid UTF-8 and / have a special meaning.
   251  		//
   252  		// The SDK can't seem to handle uploading files called '.
   253  		// - initial / encoding
   254  		// - doubled / encoding
   255  		// - trailing / encoding
   256  		// so that OSS keys are always valid file names
   257  		Default: encoder.EncodeInvalidUtf8 |
   258  			encoder.EncodeSlash |
   259  			encoder.EncodeDot,
   260  	}, {
   261  		Name: "leave_parts_on_error",
   262  		Help: `If true avoid calling abort upload on a failure, leaving all successfully uploaded parts for manual recovery.
   263  
   264  It should be set to true for resuming uploads across different sessions.
   265  
   266  WARNING: Storing parts of an incomplete multipart upload counts towards space usage on object storage and will add
   267  additional costs if not cleaned up.
   268  `,
   269  		Default:  false,
   270  		Advanced: true,
   271  	}, {
   272  		Name: "attempt_resume_upload",
   273  		Help: `If true attempt to resume previously started multipart upload for the object.
   274  This will be helpful to speed up multipart transfers by resuming uploads from past session.
   275  
   276  WARNING: If chunk size differs in resumed session from past incomplete session, then the resumed multipart upload is 
   277  aborted and a new multipart upload is started with the new chunk size.
   278  
   279  The flag leave_parts_on_error must be true to resume and optimize to skip parts that were already uploaded successfully.
   280  `,
   281  		Default:  false,
   282  		Advanced: true,
   283  	}, {
   284  		Name: "no_check_bucket",
   285  		Help: `If set, don't attempt to check the bucket exists or create it.
   286  
   287  This can be useful when trying to minimise the number of transactions
   288  rclone does if you know the bucket exists already.
   289  
   290  It can also be needed if the user you are using does not have bucket
   291  creation permissions.
   292  `,
   293  		Default:  false,
   294  		Advanced: true,
   295  	}, {
   296  		Name: "sse_customer_key_file",
   297  		Help: `To use SSE-C, a file containing the base64-encoded string of the AES-256 encryption key associated
   298  with the object. Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is needed.'`,
   299  		Advanced: true,
   300  		Examples: []fs.OptionExample{{
   301  			Value: "",
   302  			Help:  "None",
   303  		}},
   304  	}, {
   305  		Name: "sse_customer_key",
   306  		Help: `To use SSE-C, the optional header that specifies the base64-encoded 256-bit encryption key to use to
   307  encrypt or  decrypt the data. Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is
   308  needed. For more information, see Using Your Own Keys for Server-Side Encryption 
   309  (https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm)`,
   310  		Advanced: true,
   311  		Examples: []fs.OptionExample{{
   312  			Value: "",
   313  			Help:  "None",
   314  		}},
   315  	}, {
   316  		Name: "sse_customer_key_sha256",
   317  		Help: `If using SSE-C, The optional header that specifies the base64-encoded SHA256 hash of the encryption
   318  key. This value is used to check the integrity of the encryption key. see Using Your Own Keys for 
   319  Server-Side Encryption (https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm).`,
   320  		Advanced: true,
   321  		Examples: []fs.OptionExample{{
   322  			Value: "",
   323  			Help:  "None",
   324  		}},
   325  	}, {
   326  		Name: "sse_kms_key_id",
   327  		Help: `if using your own master key in vault, this header specifies the
   328  OCID (https://docs.cloud.oracle.com/Content/General/Concepts/identifiers.htm) of a master encryption key used to call
   329  the Key Management service to generate a data encryption key or to encrypt or decrypt a data encryption key.
   330  Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is needed.`,
   331  		Advanced: true,
   332  		Examples: []fs.OptionExample{{
   333  			Value: "",
   334  			Help:  "None",
   335  		}},
   336  	}, {
   337  		Name: "sse_customer_algorithm",
   338  		Help: `If using SSE-C, the optional header that specifies "AES256" as the encryption algorithm.
   339  Object Storage supports "AES256" as the encryption algorithm. For more information, see
   340  Using Your Own Keys for Server-Side Encryption (https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm).`,
   341  		Advanced: true,
   342  		Examples: []fs.OptionExample{{
   343  			Value: "",
   344  			Help:  "None",
   345  		}, {
   346  			Value: sseDefaultAlgorithm,
   347  			Help:  sseDefaultAlgorithm,
   348  		}},
   349  	}}
   350  }