github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/drive/metadata.go (about) 1 package drive 2 3 import ( 4 "context" 5 "encoding/json" 6 "fmt" 7 "strconv" 8 "strings" 9 "sync" 10 11 "github.com/rclone/rclone/fs" 12 "github.com/rclone/rclone/fs/fserrors" 13 "github.com/rclone/rclone/lib/errcount" 14 "golang.org/x/sync/errgroup" 15 drive "google.golang.org/api/drive/v3" 16 "google.golang.org/api/googleapi" 17 ) 18 19 // system metadata keys which this backend owns 20 var systemMetadataInfo = map[string]fs.MetadataHelp{ 21 "content-type": { 22 Help: "The MIME type of the file.", 23 Type: "string", 24 Example: "text/plain", 25 }, 26 "mtime": { 27 Help: "Time of last modification with mS accuracy.", 28 Type: "RFC 3339", 29 Example: "2006-01-02T15:04:05.999Z07:00", 30 }, 31 "btime": { 32 Help: "Time of file birth (creation) with mS accuracy. Note that this is only writable on fresh uploads - it can't be written for updates.", 33 Type: "RFC 3339", 34 Example: "2006-01-02T15:04:05.999Z07:00", 35 }, 36 "copy-requires-writer-permission": { 37 Help: "Whether the options to copy, print, or download this file, should be disabled for readers and commenters.", 38 Type: "boolean", 39 Example: "true", 40 }, 41 "writers-can-share": { 42 Help: "Whether users with only writer permission can modify the file's permissions. Not populated and ignored when setting for items in shared drives.", 43 Type: "boolean", 44 Example: "false", 45 }, 46 "viewed-by-me": { 47 Help: "Whether the file has been viewed by this user.", 48 Type: "boolean", 49 Example: "true", 50 ReadOnly: true, 51 }, 52 "owner": { 53 Help: "The owner of the file. Usually an email address. Enable with --drive-metadata-owner.", 54 Type: "string", 55 Example: "user@example.com", 56 }, 57 "permissions": { 58 Help: "Permissions in a JSON dump of Google drive format. On shared drives these will only be present if they aren't inherited. Enable with --drive-metadata-permissions.", 59 Type: "JSON", 60 Example: "{}", 61 }, 62 "folder-color-rgb": { 63 Help: "The color for a folder or a shortcut to a folder as an RGB hex string.", 64 Type: "string", 65 Example: "881133", 66 }, 67 "description": { 68 Help: "A short description of the file.", 69 Type: "string", 70 Example: "Contract for signing", 71 }, 72 "starred": { 73 Help: "Whether the user has starred the file.", 74 Type: "boolean", 75 Example: "false", 76 }, 77 "labels": { 78 Help: "Labels attached to this file in a JSON dump of Googled drive format. Enable with --drive-metadata-labels.", 79 Type: "JSON", 80 Example: "[]", 81 }, 82 } 83 84 // Extra fields we need to fetch to implement the system metadata above 85 var metadataFields = googleapi.Field(strings.Join([]string{ 86 "copyRequiresWriterPermission", 87 "description", 88 "folderColorRgb", 89 "hasAugmentedPermissions", 90 "owners", 91 "permissionIds", 92 "permissions", 93 "properties", 94 "starred", 95 "viewedByMe", 96 "viewedByMeTime", 97 "writersCanShare", 98 }, ",")) 99 100 // Fields we need to read from permissions 101 var permissionsFields = googleapi.Field(strings.Join([]string{ 102 "*", 103 "permissionDetails/*", 104 }, ",")) 105 106 // getPermission returns permissions for the fileID and permissionID passed in 107 func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, useCache bool) (perm *drive.Permission, inherited bool, err error) { 108 f.permissionsMu.Lock() 109 defer f.permissionsMu.Unlock() 110 if useCache { 111 perm = f.permissions[permissionID] 112 if perm != nil { 113 return perm, false, nil 114 } 115 } 116 fs.Debugf(f, "Fetching permission %q", permissionID) 117 err = f.pacer.Call(func() (bool, error) { 118 perm, err = f.svc.Permissions.Get(fileID, permissionID). 119 Fields(permissionsFields). 120 SupportsAllDrives(true). 121 Context(ctx).Do() 122 return f.shouldRetry(ctx, err) 123 }) 124 if err != nil { 125 return nil, false, err 126 } 127 128 inherited = len(perm.PermissionDetails) > 0 && perm.PermissionDetails[0].Inherited 129 130 cleanPermission(perm) 131 132 // cache the permission 133 f.permissions[permissionID] = perm 134 135 return perm, inherited, err 136 } 137 138 // Set the permissions on the info 139 func (f *Fs) setPermissions(ctx context.Context, info *drive.File, permissions []*drive.Permission) (err error) { 140 errs := errcount.New() 141 for _, perm := range permissions { 142 if perm.Role == "owner" { 143 // ignore owner permissions - these are set with owner 144 continue 145 } 146 cleanPermissionForWrite(perm) 147 err := f.pacer.Call(func() (bool, error) { 148 _, err := f.svc.Permissions.Create(info.Id, perm). 149 SupportsAllDrives(true). 150 SendNotificationEmail(false). 151 Context(ctx).Do() 152 return f.shouldRetry(ctx, err) 153 }) 154 if err != nil { 155 fs.Errorf(f, "Failed to set permission %s for %q: %v", perm.Role, perm.EmailAddress, err) 156 errs.Add(err) 157 } 158 } 159 err = errs.Err("failed to set permission") 160 if err != nil { 161 err = fserrors.NoRetryError(err) 162 } 163 return err 164 } 165 166 // Clean attributes from permissions which we can't write 167 func cleanPermissionForWrite(perm *drive.Permission) { 168 perm.Deleted = false 169 perm.DisplayName = "" 170 perm.Id = "" 171 perm.Kind = "" 172 perm.PermissionDetails = nil 173 perm.TeamDrivePermissionDetails = nil 174 } 175 176 // Clean and cache the permission if not already cached 177 func (f *Fs) cleanAndCachePermission(perm *drive.Permission) { 178 f.permissionsMu.Lock() 179 defer f.permissionsMu.Unlock() 180 cleanPermission(perm) 181 if _, found := f.permissions[perm.Id]; !found { 182 f.permissions[perm.Id] = perm 183 } 184 } 185 186 // Clean fields we don't need to keep from the permission 187 func cleanPermission(perm *drive.Permission) { 188 // DisplayName: Output only. The "pretty" name of the value of the 189 // permission. The following is a list of examples for each type of 190 // permission: * `user` - User's full name, as defined for their Google 191 // account, such as "Joe Smith." * `group` - Name of the Google Group, 192 // such as "The Company Administrators." * `domain` - String domain 193 // name, such as "thecompany.com." * `anyone` - No `displayName` is 194 // present. 195 perm.DisplayName = "" 196 197 // Kind: Output only. Identifies what kind of resource this is. Value: 198 // the fixed string "drive#permission". 199 perm.Kind = "" 200 201 // PermissionDetails: Output only. Details of whether the permissions on 202 // this shared drive item are inherited or directly on this item. This 203 // is an output-only field which is present only for shared drive items. 204 perm.PermissionDetails = nil 205 206 // PhotoLink: Output only. A link to the user's profile photo, if 207 // available. 208 perm.PhotoLink = "" 209 210 // TeamDrivePermissionDetails: Output only. Deprecated: Output only. Use 211 // `permissionDetails` instead. 212 perm.TeamDrivePermissionDetails = nil 213 } 214 215 // Fields we need to read from labels 216 var labelsFields = googleapi.Field(strings.Join([]string{ 217 "*", 218 }, ",")) 219 220 // getLabels returns labels for the fileID passed in 221 func (f *Fs) getLabels(ctx context.Context, fileID string) (labels []*drive.Label, err error) { 222 fs.Debugf(f, "Fetching labels for %q", fileID) 223 listLabels := f.svc.Files.ListLabels(fileID). 224 Fields(labelsFields). 225 Context(ctx) 226 for { 227 var info *drive.LabelList 228 err = f.pacer.Call(func() (bool, error) { 229 info, err = listLabels.Do() 230 return f.shouldRetry(ctx, err) 231 }) 232 if err != nil { 233 return nil, err 234 } 235 labels = append(labels, info.Labels...) 236 if info.NextPageToken == "" { 237 break 238 } 239 listLabels.PageToken(info.NextPageToken) 240 } 241 for _, label := range labels { 242 cleanLabel(label) 243 } 244 return labels, nil 245 } 246 247 // Set the labels on the info 248 func (f *Fs) setLabels(ctx context.Context, info *drive.File, labels []*drive.Label) (err error) { 249 if len(labels) == 0 { 250 return nil 251 } 252 req := drive.ModifyLabelsRequest{} 253 for _, label := range labels { 254 req.LabelModifications = append(req.LabelModifications, &drive.LabelModification{ 255 FieldModifications: labelFieldsToFieldModifications(label.Fields), 256 LabelId: label.Id, 257 }) 258 } 259 err = f.pacer.Call(func() (bool, error) { 260 _, err = f.svc.Files.ModifyLabels(info.Id, &req). 261 Context(ctx).Do() 262 return f.shouldRetry(ctx, err) 263 }) 264 if err != nil { 265 return fmt.Errorf("failed to set labels: %w", err) 266 } 267 return nil 268 } 269 270 // Convert label fields into something which can set the fields 271 func labelFieldsToFieldModifications(fields map[string]drive.LabelField) (out []*drive.LabelFieldModification) { 272 for id, field := range fields { 273 var emails []string 274 for _, user := range field.User { 275 emails = append(emails, user.EmailAddress) 276 } 277 out = append(out, &drive.LabelFieldModification{ 278 // FieldId: The ID of the field to be modified. 279 FieldId: id, 280 281 // SetDateValues: Replaces the value of a dateString Field with these 282 // new values. The string must be in the RFC 3339 full-date format: 283 // YYYY-MM-DD. 284 SetDateValues: field.DateString, 285 286 // SetIntegerValues: Replaces the value of an `integer` field with these 287 // new values. 288 SetIntegerValues: field.Integer, 289 290 // SetSelectionValues: Replaces a `selection` field with these new 291 // values. 292 SetSelectionValues: field.Selection, 293 294 // SetTextValues: Sets the value of a `text` field. 295 SetTextValues: field.Text, 296 297 // SetUserValues: Replaces a `user` field with these new values. The 298 // values must be valid email addresses. 299 SetUserValues: emails, 300 }) 301 } 302 return out 303 } 304 305 // Clean fields we don't need to keep from the label 306 func cleanLabel(label *drive.Label) { 307 // Kind: This is always drive#label 308 label.Kind = "" 309 310 for name, field := range label.Fields { 311 // Kind: This is always drive#labelField. 312 field.Kind = "" 313 314 // Note the fields are copies so we need to write them 315 // back to the map 316 label.Fields[name] = field 317 } 318 } 319 320 // Parse the metadata from drive item 321 // 322 // It should return nil if there is no Metadata 323 func (o *baseObject) parseMetadata(ctx context.Context, info *drive.File) (err error) { 324 metadata := make(fs.Metadata, 16) 325 326 // Dump user metadata first as it overrides system metadata 327 for k, v := range info.Properties { 328 metadata[k] = v 329 } 330 331 // System metadata 332 metadata["copy-requires-writer-permission"] = fmt.Sprint(info.CopyRequiresWriterPermission) 333 metadata["writers-can-share"] = fmt.Sprint(info.WritersCanShare) 334 metadata["viewed-by-me"] = fmt.Sprint(info.ViewedByMe) 335 metadata["content-type"] = info.MimeType 336 337 // Owners: Output only. The owner of this file. Only certain legacy 338 // files may have more than one owner. This field isn't populated for 339 // items in shared drives. 340 if o.fs.opt.MetadataOwner.IsSet(rwRead) && len(info.Owners) > 0 { 341 user := info.Owners[0] 342 if len(info.Owners) > 1 { 343 fs.Logf(o, "Ignoring more than 1 owner") 344 } 345 if user != nil { 346 id := user.EmailAddress 347 if id == "" { 348 id = user.DisplayName 349 } 350 metadata["owner"] = id 351 } 352 } 353 354 if o.fs.opt.MetadataPermissions.IsSet(rwRead) { 355 // We only write permissions out if they are not inherited. 356 // 357 // On My Drives permissions seem to be attached to every item 358 // so they will always be written out. 359 // 360 // On Shared Drives only non-inherited permissions will be 361 // written out. 362 363 // To read the inherited permissions flag will mean we need to 364 // read the permissions for each object and the cache will be 365 // useless. However shared drives don't return permissions 366 // only permissionIds so will need to fetch them for each 367 // object. We use HasAugmentedPermissions to see if there are 368 // special permissions before fetching them to save transactions. 369 370 // HasAugmentedPermissions: Output only. Whether there are permissions 371 // directly on this file. This field is only populated for items in 372 // shared drives. 373 if o.fs.isTeamDrive && !info.HasAugmentedPermissions { 374 // Don't process permissions if there aren't any specifically set 375 info.Permissions = nil 376 info.PermissionIds = nil 377 } 378 379 // PermissionIds: Output only. List of permission IDs for users with 380 // access to this file. 381 // 382 // Only process these if we have no Permissions 383 if len(info.PermissionIds) > 0 && len(info.Permissions) == 0 { 384 info.Permissions = make([]*drive.Permission, 0, len(info.PermissionIds)) 385 g, gCtx := errgroup.WithContext(ctx) 386 g.SetLimit(o.fs.ci.Checkers) 387 var mu sync.Mutex // protect the info.Permissions from concurrent writes 388 for _, permissionID := range info.PermissionIds { 389 permissionID := permissionID 390 g.Go(func() error { 391 // must fetch the team drive ones individually to check the inherited flag 392 perm, inherited, err := o.fs.getPermission(gCtx, actualID(info.Id), permissionID, !o.fs.isTeamDrive) 393 if err != nil { 394 return fmt.Errorf("failed to read permission: %w", err) 395 } 396 // Don't write inherited permissions out 397 if inherited { 398 return nil 399 } 400 // Don't write owner role out - these are covered by the owner metadata 401 if perm.Role == "owner" { 402 return nil 403 } 404 mu.Lock() 405 info.Permissions = append(info.Permissions, perm) 406 mu.Unlock() 407 return nil 408 }) 409 } 410 err = g.Wait() 411 if err != nil { 412 return err 413 } 414 } else { 415 // Clean the fetched permissions 416 for _, perm := range info.Permissions { 417 o.fs.cleanAndCachePermission(perm) 418 } 419 } 420 421 // Permissions: Output only. The full list of permissions for the file. 422 // This is only available if the requesting user can share the file. Not 423 // populated for items in shared drives. 424 if len(info.Permissions) > 0 { 425 buf, err := json.Marshal(info.Permissions) 426 if err != nil { 427 return fmt.Errorf("failed to marshal permissions: %w", err) 428 } 429 metadata["permissions"] = string(buf) 430 } 431 432 // Permission propagation 433 // https://developers.google.com/drive/api/guides/manage-sharing#permission-propagation 434 // Leads me to believe that in non shared drives, permissions 435 // are added to each item when you set permissions for a 436 // folder whereas in shared drives they are inherited and 437 // placed on the item directly. 438 } 439 440 if info.FolderColorRgb != "" { 441 metadata["folder-color-rgb"] = info.FolderColorRgb 442 } 443 if info.Description != "" { 444 metadata["description"] = info.Description 445 } 446 metadata["starred"] = fmt.Sprint(info.Starred) 447 metadata["btime"] = info.CreatedTime 448 metadata["mtime"] = info.ModifiedTime 449 450 if o.fs.opt.MetadataLabels.IsSet(rwRead) { 451 // FIXME would be really nice if we knew if files had labels 452 // before listing but we need to know all possible label IDs 453 // to get it in the listing. 454 455 labels, err := o.fs.getLabels(ctx, actualID(info.Id)) 456 if err != nil { 457 return fmt.Errorf("failed to fetch labels: %w", err) 458 } 459 buf, err := json.Marshal(labels) 460 if err != nil { 461 return fmt.Errorf("failed to marshal labels: %w", err) 462 } 463 metadata["labels"] = string(buf) 464 } 465 466 o.metadata = &metadata 467 return nil 468 } 469 470 // Set the owner on the info 471 func (f *Fs) setOwner(ctx context.Context, info *drive.File, owner string) (err error) { 472 perm := drive.Permission{ 473 Role: "owner", 474 EmailAddress: owner, 475 // Type: The type of the grantee. Valid values are: * `user` * `group` * 476 // `domain` * `anyone` When creating a permission, if `type` is `user` 477 // or `group`, you must provide an `emailAddress` for the user or group. 478 // When `type` is `domain`, you must provide a `domain`. There isn't 479 // extra information required for an `anyone` type. 480 Type: "user", 481 } 482 err = f.pacer.Call(func() (bool, error) { 483 _, err = f.svc.Permissions.Create(info.Id, &perm). 484 SupportsAllDrives(true). 485 TransferOwnership(true). 486 // SendNotificationEmail(false). - required apparently! 487 Context(ctx).Do() 488 return f.shouldRetry(ctx, err) 489 }) 490 if err != nil { 491 return fmt.Errorf("failed to set owner: %w", err) 492 } 493 return nil 494 } 495 496 // Call back to set metadata that can't be set on the upload/update 497 // 498 // The *drive.File passed in holds the current state of the drive.File 499 // and this should update it with any modifications. 500 type updateMetadataFn func(context.Context, *drive.File) error 501 502 // read the metadata from meta and write it into updateInfo 503 // 504 // update should be true if this is being used to create metadata for 505 // an update/PATCH call as the rules on what can be updated are 506 // slightly different there. 507 // 508 // It returns a callback which should be called to finish the updates 509 // after the data is uploaded. 510 func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs.Metadata, update bool) (callback updateMetadataFn, err error) { 511 callbackFns := []updateMetadataFn{} 512 callback = func(ctx context.Context, info *drive.File) error { 513 for _, fn := range callbackFns { 514 err := fn(ctx, info) 515 if err != nil { 516 return err 517 } 518 } 519 return nil 520 } 521 // merge metadata into request and user metadata 522 for k, v := range meta { 523 k, v := k, v 524 // parse a boolean from v and write into out 525 parseBool := func(out *bool) error { 526 b, err := strconv.ParseBool(v) 527 if err != nil { 528 return fmt.Errorf("can't parse metadata %q = %q: %w", k, v, err) 529 } 530 *out = b 531 return nil 532 } 533 switch k { 534 case "copy-requires-writer-permission": 535 if err := parseBool(&updateInfo.CopyRequiresWriterPermission); err != nil { 536 return nil, err 537 } 538 case "writers-can-share": 539 if !f.isTeamDrive { 540 if err := parseBool(&updateInfo.WritersCanShare); err != nil { 541 return nil, err 542 } 543 } else { 544 fs.Debugf(f, "Ignoring %s=%s as can't set on shared drives", k, v) 545 } 546 case "viewed-by-me": 547 // Can't write this 548 case "content-type": 549 updateInfo.MimeType = v 550 case "owner": 551 if !f.opt.MetadataOwner.IsSet(rwWrite) { 552 continue 553 } 554 // Can't set Owner on upload so need to set afterwards 555 callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error { 556 err := f.setOwner(ctx, info, v) 557 if err != nil && f.opt.MetadataOwner.IsSet(rwFailOK) { 558 fs.Errorf(f, "Ignoring error as failok is set: %v", err) 559 return nil 560 } 561 return err 562 }) 563 case "permissions": 564 if !f.opt.MetadataPermissions.IsSet(rwWrite) { 565 continue 566 } 567 var perms []*drive.Permission 568 err := json.Unmarshal([]byte(v), &perms) 569 if err != nil { 570 return nil, fmt.Errorf("failed to unmarshal permissions: %w", err) 571 } 572 // Can't set Permissions on upload so need to set afterwards 573 callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error { 574 err := f.setPermissions(ctx, info, perms) 575 if err != nil && f.opt.MetadataPermissions.IsSet(rwFailOK) { 576 // We've already logged the permissions errors individually here 577 fs.Debugf(f, "Ignoring error as failok is set: %v", err) 578 return nil 579 } 580 return err 581 }) 582 case "labels": 583 if !f.opt.MetadataLabels.IsSet(rwWrite) { 584 continue 585 } 586 var labels []*drive.Label 587 err := json.Unmarshal([]byte(v), &labels) 588 if err != nil { 589 return nil, fmt.Errorf("failed to unmarshal labels: %w", err) 590 } 591 // Can't set Labels on upload so need to set afterwards 592 callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error { 593 err := f.setLabels(ctx, info, labels) 594 if err != nil && f.opt.MetadataLabels.IsSet(rwFailOK) { 595 fs.Errorf(f, "Ignoring error as failok is set: %v", err) 596 return nil 597 } 598 return err 599 }) 600 case "folder-color-rgb": 601 updateInfo.FolderColorRgb = v 602 case "description": 603 updateInfo.Description = v 604 case "starred": 605 if err := parseBool(&updateInfo.Starred); err != nil { 606 return nil, err 607 } 608 case "btime": 609 if update { 610 fs.Debugf(f, "Skipping btime metadata as can't update it on an existing file: %v", v) 611 } else { 612 updateInfo.CreatedTime = v 613 } 614 case "mtime": 615 updateInfo.ModifiedTime = v 616 default: 617 if updateInfo.Properties == nil { 618 updateInfo.Properties = make(map[string]string, 1) 619 } 620 updateInfo.Properties[k] = v 621 } 622 } 623 return callback, nil 624 } 625 626 // Fetch metadata and update updateInfo if --metadata is in use 627 func (f *Fs) fetchAndUpdateMetadata(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption, updateInfo *drive.File, update bool) (callback updateMetadataFn, err error) { 628 meta, err := fs.GetMetadataOptions(ctx, f, src, options) 629 if err != nil { 630 return nil, fmt.Errorf("failed to read metadata from source object: %w", err) 631 } 632 callback, err = f.updateMetadata(ctx, updateInfo, meta, update) 633 if err != nil { 634 return nil, fmt.Errorf("failed to update metadata from source object: %w", err) 635 } 636 return callback, nil 637 }