github.com/artpar/rclone@v1.67.3/backend/oracleobjectstorage/object.go (about) 1 //go:build !plan9 && !solaris && !js 2 3 package oracleobjectstorage 4 5 import ( 6 "bytes" 7 "context" 8 "encoding/base64" 9 "encoding/hex" 10 "fmt" 11 "io" 12 "net/http" 13 "os" 14 "regexp" 15 "strconv" 16 "strings" 17 "time" 18 19 "github.com/artpar/rclone/fs" 20 "github.com/artpar/rclone/fs/hash" 21 "github.com/ncw/swift/v2" 22 "github.com/oracle/oci-go-sdk/v65/common" 23 "github.com/oracle/oci-go-sdk/v65/objectstorage" 24 ) 25 26 // ------------------------------------------------------------ 27 // Object Interface Implementation 28 // ------------------------------------------------------------ 29 30 const ( 31 metaMtime = "mtime" // the meta key to store mtime in - e.g. X-Amz-Meta-Mtime 32 metaMD5Hash = "md5chksum" // the meta key to store md5hash in 33 // StandardTier object storage tier 34 ociMetaPrefix = "opc-meta-" 35 ) 36 37 var archive = "archive" 38 var infrequentAccess = "infrequentaccess" 39 var standard = "standard" 40 41 var storageTierMap = map[string]*string{ 42 archive: &archive, 43 infrequentAccess: &infrequentAccess, 44 standard: &standard, 45 } 46 47 var matchMd5 = regexp.MustCompile(`^[0-9a-f]{32}$`) 48 49 // Object describes a oci bucket object 50 type Object struct { 51 fs *Fs // what this object is part of 52 remote string // The remote path 53 md5 string // MD5 hash if known 54 bytes int64 // Size of the object 55 lastModified time.Time // The modified time of the object if known 56 meta map[string]string // The object metadata if known - may be nil 57 mimeType string // Content-Type of the object 58 59 // Metadata as pointers to strings as they often won't be present 60 storageTier *string // e.g. Standard 61 } 62 63 // split returns bucket and bucketPath from the object 64 func (o *Object) split() (bucket, bucketPath string) { 65 return o.fs.split(o.remote) 66 } 67 68 // readMetaData gets the metadata if it hasn't already been fetched 69 func (o *Object) readMetaData(ctx context.Context) (err error) { 70 fs.Debugf(o, "trying to read metadata %v", o.remote) 71 if o.meta != nil { 72 return nil 73 } 74 info, err := o.headObject(ctx) 75 if err != nil { 76 return err 77 } 78 return o.decodeMetaDataHead(info) 79 } 80 81 // headObject gets the metadata from the object unconditionally 82 func (o *Object) headObject(ctx context.Context) (info *objectstorage.HeadObjectResponse, err error) { 83 bucketName, objectPath := o.split() 84 req := objectstorage.HeadObjectRequest{ 85 NamespaceName: common.String(o.fs.opt.Namespace), 86 BucketName: common.String(bucketName), 87 ObjectName: common.String(objectPath), 88 } 89 useBYOKHeadObject(o.fs, &req) 90 var response objectstorage.HeadObjectResponse 91 err = o.fs.pacer.Call(func() (bool, error) { 92 var err error 93 response, err = o.fs.srv.HeadObject(ctx, req) 94 return shouldRetry(ctx, response.HTTPResponse(), err) 95 }) 96 if err != nil { 97 if svcErr, ok := err.(common.ServiceError); ok { 98 if svcErr.GetHTTPStatusCode() == http.StatusNotFound { 99 return nil, fs.ErrorObjectNotFound 100 } 101 } 102 fs.Errorf(o, "Failed to head object: %v", err) 103 return nil, err 104 } 105 o.fs.cache.MarkOK(bucketName) 106 return &response, err 107 } 108 109 func (o *Object) decodeMetaDataHead(info *objectstorage.HeadObjectResponse) (err error) { 110 return o.setMetaData( 111 info.ContentLength, 112 info.ContentMd5, 113 info.ContentType, 114 info.LastModified, 115 info.StorageTier, 116 info.OpcMeta) 117 } 118 119 func (o *Object) decodeMetaDataObject(info *objectstorage.GetObjectResponse) (err error) { 120 return o.setMetaData( 121 info.ContentLength, 122 info.ContentMd5, 123 info.ContentType, 124 info.LastModified, 125 info.StorageTier, 126 info.OpcMeta) 127 } 128 129 func (o *Object) setMetaData( 130 contentLength *int64, 131 contentMd5 *string, 132 contentType *string, 133 lastModified *common.SDKTime, 134 storageTier interface{}, 135 meta map[string]string) error { 136 137 if contentLength != nil { 138 o.bytes = *contentLength 139 } 140 if contentMd5 != nil { 141 md5, err := o.base64ToMd5(*contentMd5) 142 if err == nil { 143 o.md5 = md5 144 } 145 } 146 o.meta = meta 147 if o.meta == nil { 148 o.meta = map[string]string{} 149 } 150 // Read MD5 from metadata if present 151 if md5sumBase64, ok := o.meta[metaMD5Hash]; ok { 152 md5, err := o.base64ToMd5(md5sumBase64) 153 if err != nil { 154 o.md5 = md5 155 } 156 } 157 if lastModified == nil { 158 o.lastModified = time.Now() 159 fs.Logf(o, "Failed to read last modified") 160 } else { 161 o.lastModified = lastModified.Time 162 } 163 if contentType != nil { 164 o.mimeType = *contentType 165 } 166 if storageTier == nil || storageTier == "" { 167 o.storageTier = storageTierMap[standard] 168 } else { 169 tier := strings.ToLower(fmt.Sprintf("%v", storageTier)) 170 o.storageTier = storageTierMap[tier] 171 } 172 return nil 173 } 174 175 func (o *Object) base64ToMd5(md5sumBase64 string) (md5 string, err error) { 176 md5sumBytes, err := base64.StdEncoding.DecodeString(md5sumBase64) 177 if err != nil { 178 fs.Debugf(o, "Failed to read md5sum from metadata %q: %v", md5sumBase64, err) 179 return "", err 180 } else if len(md5sumBytes) != 16 { 181 fs.Debugf(o, "failed to read md5sum from metadata %q: wrong length", md5sumBase64) 182 return "", fmt.Errorf("failed to read md5sum from metadata %q: wrong length", md5sumBase64) 183 } 184 return hex.EncodeToString(md5sumBytes), nil 185 } 186 187 // Fs returns the parent Fs 188 func (o *Object) Fs() fs.Info { 189 return o.fs 190 } 191 192 // Remote returns the remote path 193 func (o *Object) Remote() string { 194 return o.remote 195 } 196 197 // Return a string version 198 func (o *Object) String() string { 199 if o == nil { 200 return "<nil>" 201 } 202 return o.remote 203 } 204 205 // Size returns the size of an object in bytes 206 func (o *Object) Size() int64 { 207 return o.bytes 208 } 209 210 // GetTier returns storage class as string 211 func (o *Object) GetTier() string { 212 if o.storageTier == nil || *o.storageTier == "" { 213 return standard 214 } 215 return *o.storageTier 216 } 217 218 // SetTier performs changing storage class 219 func (o *Object) SetTier(tier string) (err error) { 220 ctx := context.TODO() 221 tier = strings.ToLower(tier) 222 bucketName, bucketPath := o.split() 223 tierEnum, ok := objectstorage.GetMappingStorageTierEnum(tier) 224 if !ok { 225 return fmt.Errorf("not a valid storage tier %v ", tier) 226 } 227 228 req := objectstorage.UpdateObjectStorageTierRequest{ 229 NamespaceName: common.String(o.fs.opt.Namespace), 230 BucketName: common.String(bucketName), 231 UpdateObjectStorageTierDetails: objectstorage.UpdateObjectStorageTierDetails{ 232 ObjectName: common.String(bucketPath), 233 StorageTier: tierEnum, 234 }, 235 } 236 _, err = o.fs.srv.UpdateObjectStorageTier(ctx, req) 237 if err != nil { 238 return err 239 } 240 o.storageTier = storageTierMap[tier] 241 return err 242 } 243 244 // MimeType of an Object if known, "" otherwise 245 func (o *Object) MimeType(ctx context.Context) string { 246 err := o.readMetaData(ctx) 247 if err != nil { 248 fs.Logf(o, "Failed to read metadata: %v", err) 249 return "" 250 } 251 return o.mimeType 252 } 253 254 // Hash returns the MD5 of an object returning a lowercase hex string 255 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { 256 if t != hash.MD5 { 257 return "", hash.ErrUnsupported 258 } 259 // Convert base64 encoded md5 into lower case hex 260 if o.md5 == "" { 261 err := o.readMetaData(ctx) 262 if err != nil { 263 return "", err 264 } 265 } 266 return o.md5, nil 267 } 268 269 // ModTime returns the modification time of the object 270 // 271 // It attempts to read the objects mtime and if that isn't present the 272 // LastModified returned to the http headers 273 func (o *Object) ModTime(ctx context.Context) (result time.Time) { 274 if o.fs.ci.UseServerModTime { 275 return o.lastModified 276 } 277 err := o.readMetaData(ctx) 278 if err != nil { 279 fs.Logf(o, "Failed to read metadata: %v", err) 280 return time.Now() 281 } 282 // read mtime out of metadata if available 283 d, ok := o.meta[metaMtime] 284 if !ok || d == "" { 285 return o.lastModified 286 } 287 modTime, err := swift.FloatStringToTime(d) 288 if err != nil { 289 fs.Logf(o, "Failed to read mtime from object: %v", err) 290 return o.lastModified 291 } 292 return modTime 293 } 294 295 // SetModTime sets the modification time of the local fs object 296 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { 297 err := o.readMetaData(ctx) 298 if err != nil { 299 return err 300 } 301 o.meta[metaMtime] = swift.TimeToFloatString(modTime) 302 _, err = o.fs.Copy(ctx, o, o.remote) 303 return err 304 } 305 306 // Storable returns if this object is storable 307 func (o *Object) Storable() bool { 308 return true 309 } 310 311 // Remove an object 312 func (o *Object) Remove(ctx context.Context) error { 313 bucketName, bucketPath := o.split() 314 req := objectstorage.DeleteObjectRequest{ 315 NamespaceName: common.String(o.fs.opt.Namespace), 316 BucketName: common.String(bucketName), 317 ObjectName: common.String(bucketPath), 318 } 319 err := o.fs.pacer.Call(func() (bool, error) { 320 resp, err := o.fs.srv.DeleteObject(ctx, req) 321 return shouldRetry(ctx, resp.HTTPResponse(), err) 322 }) 323 return err 324 } 325 326 // Open object file 327 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) { 328 bucketName, bucketPath := o.split() 329 req := objectstorage.GetObjectRequest{ 330 NamespaceName: common.String(o.fs.opt.Namespace), 331 BucketName: common.String(bucketName), 332 ObjectName: common.String(bucketPath), 333 } 334 o.applyGetObjectOptions(&req, options...) 335 useBYOKGetObject(o.fs, &req) 336 var resp objectstorage.GetObjectResponse 337 err := o.fs.pacer.Call(func() (bool, error) { 338 var err error 339 resp, err = o.fs.srv.GetObject(ctx, req) 340 return shouldRetry(ctx, resp.HTTPResponse(), err) 341 }) 342 if err != nil { 343 return nil, err 344 } 345 // read size from ContentLength or ContentRange 346 bytes := resp.ContentLength 347 if resp.ContentRange != nil { 348 var contentRange = *resp.ContentRange 349 slash := strings.IndexRune(contentRange, '/') 350 if slash >= 0 { 351 i, err := strconv.ParseInt(contentRange[slash+1:], 10, 64) 352 if err == nil { 353 bytes = &i 354 } else { 355 fs.Debugf(o, "Failed to find parse integer from in %q: %v", contentRange, err) 356 } 357 } else { 358 fs.Debugf(o, "Failed to find length in %q", contentRange) 359 } 360 } 361 err = o.decodeMetaDataObject(&resp) 362 if err != nil { 363 return nil, err 364 } 365 o.bytes = *bytes 366 return resp.HTTPResponse().Body, nil 367 } 368 369 func isZeroLength(streamReader io.Reader) bool { 370 switch v := streamReader.(type) { 371 case *bytes.Buffer: 372 return v.Len() == 0 373 case *bytes.Reader: 374 return v.Len() == 0 375 case *strings.Reader: 376 return v.Len() == 0 377 case *os.File: 378 fi, err := v.Stat() 379 if err != nil { 380 return false 381 } 382 return fi.Size() == 0 383 default: 384 return false 385 } 386 } 387 388 // Update an object if it has changed 389 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { 390 bucketName, _ := o.split() 391 err = o.fs.makeBucket(ctx, bucketName) 392 if err != nil { 393 return err 394 } 395 396 // determine if we like upload single or multipart. 397 size := src.Size() 398 multipart := size < 0 || size >= int64(o.fs.opt.UploadCutoff) 399 if isZeroLength(in) { 400 multipart = false 401 } 402 if multipart { 403 err = o.uploadMultipart(ctx, src, in, options...) 404 if err != nil { 405 return err 406 } 407 } else { 408 ui, err := o.prepareUpload(ctx, src, options) 409 if err != nil { 410 return fmt.Errorf("failed to prepare upload: %w", err) 411 } 412 var resp objectstorage.PutObjectResponse 413 err = o.fs.pacer.CallNoRetry(func() (bool, error) { 414 ui.req.PutObjectBody = io.NopCloser(in) 415 resp, err = o.fs.srv.PutObject(ctx, *ui.req) 416 return shouldRetry(ctx, resp.HTTPResponse(), err) 417 }) 418 if err != nil { 419 fs.Errorf(o, "put object failed %v", err) 420 return err 421 } 422 } 423 // Read the metadata from the newly created object 424 o.meta = nil // wipe old metadata 425 return o.readMetaData(ctx) 426 } 427 428 func (o *Object) applyPutOptions(req *objectstorage.PutObjectRequest, options ...fs.OpenOption) { 429 // Apply upload options 430 for _, option := range options { 431 key, value := option.Header() 432 lowerKey := strings.ToLower(key) 433 switch lowerKey { 434 case "": 435 // ignore 436 case "cache-control": 437 req.CacheControl = common.String(value) 438 case "content-disposition": 439 req.ContentDisposition = common.String(value) 440 case "content-encoding": 441 req.ContentEncoding = common.String(value) 442 case "content-language": 443 req.ContentLanguage = common.String(value) 444 case "content-type": 445 req.ContentType = common.String(value) 446 default: 447 if strings.HasPrefix(lowerKey, ociMetaPrefix) { 448 req.OpcMeta[lowerKey] = value 449 } else { 450 fs.Errorf(o, "Don't know how to set key %q on upload", key) 451 } 452 } 453 } 454 } 455 456 func (o *Object) applyGetObjectOptions(req *objectstorage.GetObjectRequest, options ...fs.OpenOption) { 457 fs.FixRangeOption(options, o.bytes) 458 for _, option := range options { 459 switch option.(type) { 460 case *fs.RangeOption, *fs.SeekOption: 461 _, value := option.Header() 462 req.Range = &value 463 default: 464 if option.Mandatory() { 465 fs.Logf(o, "Unsupported mandatory option: %v", option) 466 } 467 } 468 } 469 // Apply upload options 470 for _, option := range options { 471 key, value := option.Header() 472 lowerKey := strings.ToLower(key) 473 switch lowerKey { 474 case "": 475 // ignore 476 case "cache-control": 477 req.HttpResponseCacheControl = common.String(value) 478 case "content-disposition": 479 req.HttpResponseContentDisposition = common.String(value) 480 case "content-encoding": 481 req.HttpResponseContentEncoding = common.String(value) 482 case "content-language": 483 req.HttpResponseContentLanguage = common.String(value) 484 case "content-type": 485 req.HttpResponseContentType = common.String(value) 486 case "range": 487 // do nothing 488 default: 489 fs.Errorf(o, "Don't know how to set key %q on upload", key) 490 } 491 } 492 } 493 494 func (o *Object) applyMultipartUploadOptions(putReq *objectstorage.PutObjectRequest, req *objectstorage.CreateMultipartUploadRequest) { 495 req.ContentType = putReq.ContentType 496 req.ContentLanguage = putReq.ContentLanguage 497 req.ContentEncoding = putReq.ContentEncoding 498 req.ContentDisposition = putReq.ContentDisposition 499 req.CacheControl = putReq.CacheControl 500 req.Metadata = metadataWithOpcPrefix(putReq.OpcMeta) 501 req.OpcSseCustomerAlgorithm = putReq.OpcSseCustomerAlgorithm 502 req.OpcSseCustomerKey = putReq.OpcSseCustomerKey 503 req.OpcSseCustomerKeySha256 = putReq.OpcSseCustomerKeySha256 504 req.OpcSseKmsKeyId = putReq.OpcSseKmsKeyId 505 } 506 507 func (o *Object) applyPartUploadOptions(putReq *objectstorage.PutObjectRequest, req *objectstorage.UploadPartRequest) { 508 req.OpcSseCustomerAlgorithm = putReq.OpcSseCustomerAlgorithm 509 req.OpcSseCustomerKey = putReq.OpcSseCustomerKey 510 req.OpcSseCustomerKeySha256 = putReq.OpcSseCustomerKeySha256 511 req.OpcSseKmsKeyId = putReq.OpcSseKmsKeyId 512 } 513 514 func metadataWithOpcPrefix(src map[string]string) map[string]string { 515 dst := make(map[string]string) 516 for lowerKey, value := range src { 517 if !strings.HasPrefix(lowerKey, ociMetaPrefix) { 518 dst[ociMetaPrefix+lowerKey] = value 519 } 520 } 521 return dst 522 }