github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/oracleobjectstorage/options.go (about) 1 //go:build !plan9 && !solaris && !js 2 3 package oracleobjectstorage 4 5 import ( 6 "time" 7 8 "github.com/rclone/rclone/fs" 9 "github.com/rclone/rclone/fs/config" 10 "github.com/rclone/rclone/lib/encoder" 11 ) 12 13 const ( 14 maxSizeForCopy = 4768 * 1024 * 1024 15 maxUploadParts = 10000 16 defaultUploadConcurrency = 10 17 minChunkSize = fs.SizeSuffix(5 * 1024 * 1024) 18 defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024) 19 maxUploadCutoff = fs.SizeSuffix(5 * 1024 * 1024 * 1024) 20 minSleep = 10 * time.Millisecond 21 defaultCopyTimeoutDuration = fs.Duration(time.Minute) 22 ) 23 24 const ( 25 userPrincipal = "user_principal_auth" 26 instancePrincipal = "instance_principal_auth" 27 resourcePrincipal = "resource_principal_auth" 28 workloadIdentity = "workload_identity_auth" 29 environmentAuth = "env_auth" 30 noAuth = "no_auth" 31 32 userPrincipalHelpText = `use an OCI user and an API key for authentication. 33 you’ll need to put in a config file your tenancy OCID, user OCID, region, the path, fingerprint to an API key. 34 https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm` 35 36 instancePrincipalHelpText = `use instance principals to authorize an instance to make API calls. 37 each instance has its own identity, and authenticates using the certificates that are read from instance metadata. 38 https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfrominstances.htm` 39 40 workloadIdentityHelpText = `use workload identity to grant OCI Container Engine for Kubernetes workloads policy-driven access to OCI resources using OCI Identity and Access Management (IAM). 41 https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contenggrantingworkloadaccesstoresources.htm` 42 resourcePrincipalHelpText = `use resource principals to make API calls` 43 44 environmentAuthHelpText = `automatically pickup the credentials from runtime(env), first one to provide auth wins` 45 46 noAuthHelpText = `no credentials needed, this is typically for reading public buckets` 47 ) 48 49 // Options defines the configuration for this backend 50 type Options struct { 51 Provider string `config:"provider"` 52 Compartment string `config:"compartment"` 53 Namespace string `config:"namespace"` 54 Region string `config:"region"` 55 Endpoint string `config:"endpoint"` 56 Enc encoder.MultiEncoder `config:"encoding"` 57 ConfigFile string `config:"config_file"` 58 ConfigProfile string `config:"config_profile"` 59 UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` 60 ChunkSize fs.SizeSuffix `config:"chunk_size"` 61 MaxUploadParts int `config:"max_upload_parts"` 62 UploadConcurrency int `config:"upload_concurrency"` 63 DisableChecksum bool `config:"disable_checksum"` 64 CopyCutoff fs.SizeSuffix `config:"copy_cutoff"` 65 CopyTimeout fs.Duration `config:"copy_timeout"` 66 StorageTier string `config:"storage_tier"` 67 LeavePartsOnError bool `config:"leave_parts_on_error"` 68 AttemptResumeUpload bool `config:"attempt_resume_upload"` 69 NoCheckBucket bool `config:"no_check_bucket"` 70 SSEKMSKeyID string `config:"sse_kms_key_id"` 71 SSECustomerAlgorithm string `config:"sse_customer_algorithm"` 72 SSECustomerKey string `config:"sse_customer_key"` 73 SSECustomerKeyFile string `config:"sse_customer_key_file"` 74 SSECustomerKeySha256 string `config:"sse_customer_key_sha256"` 75 } 76 77 func newOptions() []fs.Option { 78 return []fs.Option{{ 79 Name: fs.ConfigProvider, 80 Help: "Choose your Auth Provider", 81 Required: true, 82 Default: environmentAuth, 83 Examples: []fs.OptionExample{{ 84 Value: environmentAuth, 85 Help: environmentAuthHelpText, 86 }, { 87 Value: userPrincipal, 88 Help: userPrincipalHelpText, 89 }, { 90 Value: instancePrincipal, 91 Help: instancePrincipalHelpText, 92 }, { 93 Value: workloadIdentity, 94 Help: workloadIdentityHelpText, 95 }, { 96 Value: resourcePrincipal, 97 Help: resourcePrincipalHelpText, 98 }, { 99 Value: noAuth, 100 Help: noAuthHelpText, 101 }}, 102 }, { 103 Name: "namespace", 104 Help: "Object storage namespace", 105 Required: true, 106 Sensitive: true, 107 }, { 108 Name: "compartment", 109 Help: "Object storage compartment OCID", 110 Provider: "!no_auth", 111 Required: true, 112 Sensitive: true, 113 }, { 114 Name: "region", 115 Help: "Object storage Region", 116 Required: true, 117 }, { 118 Name: "endpoint", 119 Help: "Endpoint for Object storage API.\n\nLeave blank to use the default endpoint for the region.", 120 Required: false, 121 }, { 122 Name: "config_file", 123 Help: "Path to OCI config file", 124 Provider: userPrincipal, 125 Default: "~/.oci/config", 126 Examples: []fs.OptionExample{{ 127 Value: "~/.oci/config", 128 Help: "oci configuration file location", 129 }}, 130 }, { 131 Name: "config_profile", 132 Help: "Profile name inside the oci config file", 133 Provider: userPrincipal, 134 Default: "Default", 135 Examples: []fs.OptionExample{{ 136 Value: "Default", 137 Help: "Use the default profile", 138 }}, 139 }, { 140 // Mapping from here: https://github.com/oracle/oci-go-sdk/blob/master/objectstorage/storage_tier.go 141 Name: "storage_tier", 142 Help: "The storage class to use when storing new objects in storage. https://docs.oracle.com/en-us/iaas/Content/Object/Concepts/understandingstoragetiers.htm", 143 Default: "Standard", 144 Advanced: true, 145 Examples: []fs.OptionExample{{ 146 Value: "Standard", 147 Help: "Standard storage tier, this is the default tier", 148 }, { 149 Value: "InfrequentAccess", 150 Help: "InfrequentAccess storage tier", 151 }, { 152 Value: "Archive", 153 Help: "Archive storage tier", 154 }}, 155 }, { 156 Name: "upload_cutoff", 157 Help: `Cutoff for switching to chunked upload. 158 159 Any files larger than this will be uploaded in chunks of chunk_size. 160 The minimum is 0 and the maximum is 5 GiB.`, 161 Default: defaultUploadCutoff, 162 Advanced: true, 163 }, { 164 Name: "chunk_size", 165 Help: `Chunk size to use for uploading. 166 167 When uploading files larger than upload_cutoff or files with unknown 168 size (e.g. from "rclone rcat" or uploaded with "rclone mount" they will be uploaded 169 as multipart uploads using this chunk size. 170 171 Note that "upload_concurrency" chunks of this size are buffered 172 in memory per transfer. 173 174 If you are transferring large files over high-speed links and you have 175 enough memory, then increasing this will speed up the transfers. 176 177 Rclone will automatically increase the chunk size when uploading a 178 large file of known size to stay below the 10,000 chunks limit. 179 180 Files of unknown size are uploaded with the configured 181 chunk_size. Since the default chunk size is 5 MiB and there can be at 182 most 10,000 chunks, this means that by default the maximum size of 183 a file you can stream upload is 48 GiB. If you wish to stream upload 184 larger files then you will need to increase chunk_size. 185 186 Increasing the chunk size decreases the accuracy of the progress 187 statistics displayed with "-P" flag. 188 `, 189 Default: minChunkSize, 190 Advanced: true, 191 }, { 192 Name: "max_upload_parts", 193 Help: `Maximum number of parts in a multipart upload. 194 195 This option defines the maximum number of multipart chunks to use 196 when doing a multipart upload. 197 198 OCI has max parts limit of 10,000 chunks. 199 200 Rclone will automatically increase the chunk size when uploading a 201 large file of a known size to stay below this number of chunks limit. 202 `, 203 Default: maxUploadParts, 204 Advanced: true, 205 }, { 206 Name: "upload_concurrency", 207 Help: `Concurrency for multipart uploads. 208 209 This is the number of chunks of the same file that are uploaded 210 concurrently. 211 212 If you are uploading small numbers of large files over high-speed links 213 and these uploads do not fully utilize your bandwidth, then increasing 214 this may help to speed up the transfers.`, 215 Default: defaultUploadConcurrency, 216 Advanced: true, 217 }, { 218 Name: "copy_cutoff", 219 Help: `Cutoff for switching to multipart copy. 220 221 Any files larger than this that need to be server-side copied will be 222 copied in chunks of this size. 223 224 The minimum is 0 and the maximum is 5 GiB.`, 225 Default: fs.SizeSuffix(maxSizeForCopy), 226 Advanced: true, 227 }, { 228 Name: "copy_timeout", 229 Help: `Timeout for copy. 230 231 Copy is an asynchronous operation, specify timeout to wait for copy to succeed 232 `, 233 Default: defaultCopyTimeoutDuration, 234 Advanced: true, 235 }, { 236 Name: "disable_checksum", 237 Help: `Don't store MD5 checksum with object metadata. 238 239 Normally rclone will calculate the MD5 checksum of the input before 240 uploading it so it can add it to metadata on the object. This is great 241 for data integrity checking but can cause long delays for large files 242 to start uploading.`, 243 Default: false, 244 Advanced: true, 245 }, { 246 Name: config.ConfigEncoding, 247 Help: config.ConfigEncodingHelp, 248 Advanced: true, 249 // Any UTF-8 character is valid in a key, however it can't handle 250 // invalid UTF-8 and / have a special meaning. 251 // 252 // The SDK can't seem to handle uploading files called '. 253 // - initial / encoding 254 // - doubled / encoding 255 // - trailing / encoding 256 // so that OSS keys are always valid file names 257 Default: encoder.EncodeInvalidUtf8 | 258 encoder.EncodeSlash | 259 encoder.EncodeDot, 260 }, { 261 Name: "leave_parts_on_error", 262 Help: `If true avoid calling abort upload on a failure, leaving all successfully uploaded parts for manual recovery. 263 264 It should be set to true for resuming uploads across different sessions. 265 266 WARNING: Storing parts of an incomplete multipart upload counts towards space usage on object storage and will add 267 additional costs if not cleaned up. 268 `, 269 Default: false, 270 Advanced: true, 271 }, { 272 Name: "attempt_resume_upload", 273 Help: `If true attempt to resume previously started multipart upload for the object. 274 This will be helpful to speed up multipart transfers by resuming uploads from past session. 275 276 WARNING: If chunk size differs in resumed session from past incomplete session, then the resumed multipart upload is 277 aborted and a new multipart upload is started with the new chunk size. 278 279 The flag leave_parts_on_error must be true to resume and optimize to skip parts that were already uploaded successfully. 280 `, 281 Default: false, 282 Advanced: true, 283 }, { 284 Name: "no_check_bucket", 285 Help: `If set, don't attempt to check the bucket exists or create it. 286 287 This can be useful when trying to minimise the number of transactions 288 rclone does if you know the bucket exists already. 289 290 It can also be needed if the user you are using does not have bucket 291 creation permissions. 292 `, 293 Default: false, 294 Advanced: true, 295 }, { 296 Name: "sse_customer_key_file", 297 Help: `To use SSE-C, a file containing the base64-encoded string of the AES-256 encryption key associated 298 with the object. Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is needed.'`, 299 Advanced: true, 300 Examples: []fs.OptionExample{{ 301 Value: "", 302 Help: "None", 303 }}, 304 }, { 305 Name: "sse_customer_key", 306 Help: `To use SSE-C, the optional header that specifies the base64-encoded 256-bit encryption key to use to 307 encrypt or decrypt the data. Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is 308 needed. For more information, see Using Your Own Keys for Server-Side Encryption 309 (https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm)`, 310 Advanced: true, 311 Examples: []fs.OptionExample{{ 312 Value: "", 313 Help: "None", 314 }}, 315 }, { 316 Name: "sse_customer_key_sha256", 317 Help: `If using SSE-C, The optional header that specifies the base64-encoded SHA256 hash of the encryption 318 key. This value is used to check the integrity of the encryption key. see Using Your Own Keys for 319 Server-Side Encryption (https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm).`, 320 Advanced: true, 321 Examples: []fs.OptionExample{{ 322 Value: "", 323 Help: "None", 324 }}, 325 }, { 326 Name: "sse_kms_key_id", 327 Help: `if using your own master key in vault, this header specifies the 328 OCID (https://docs.cloud.oracle.com/Content/General/Concepts/identifiers.htm) of a master encryption key used to call 329 the Key Management service to generate a data encryption key or to encrypt or decrypt a data encryption key. 330 Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is needed.`, 331 Advanced: true, 332 Examples: []fs.OptionExample{{ 333 Value: "", 334 Help: "None", 335 }}, 336 }, { 337 Name: "sse_customer_algorithm", 338 Help: `If using SSE-C, the optional header that specifies "AES256" as the encryption algorithm. 339 Object Storage supports "AES256" as the encryption algorithm. For more information, see 340 Using Your Own Keys for Server-Side Encryption (https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm).`, 341 Advanced: true, 342 Examples: []fs.OptionExample{{ 343 Value: "", 344 Help: "None", 345 }, { 346 Value: sseDefaultAlgorithm, 347 Help: sseDefaultAlgorithm, 348 }}, 349 }} 350 }