github.com/keybase/client/go@v0.0.0-20241007131713-f10651d043c8/chat/attachments/uploader.go (about) 1 package attachments 2 3 import ( 4 "fmt" 5 "io" 6 "os" 7 "path/filepath" 8 "sync" 9 "time" 10 11 disklru "github.com/keybase/client/go/lru" 12 13 "github.com/keybase/client/go/encrypteddb" 14 15 "github.com/keybase/client/go/chat/globals" 16 "github.com/keybase/client/go/chat/s3" 17 "github.com/keybase/client/go/chat/storage" 18 "github.com/keybase/client/go/chat/types" 19 "github.com/keybase/client/go/chat/utils" 20 "github.com/keybase/client/go/libkb" 21 "github.com/keybase/client/go/protocol/chat1" 22 "github.com/keybase/client/go/protocol/gregor1" 23 "golang.org/x/net/context" 24 "golang.org/x/sync/errgroup" 25 ) 26 27 const ( 28 uploadedPreviewsDir = "uploadedpreviews" 29 uploadedFullsDir = "uploadedfulls" 30 uploadedTempsDir = "uploadtemps" 31 ) 32 33 type uploaderTask struct { 34 UID gregor1.UID 35 OutboxID chat1.OutboxID 36 ConvID chat1.ConversationID 37 Title, Filename string 38 Metadata []byte 39 CallerPreview *chat1.MakePreviewRes 40 } 41 42 type uploaderStatus struct { 43 Status types.AttachmentUploaderTaskStatus 44 Result types.AttachmentUploadResult 45 } 46 47 type uploaderResult struct { 48 sync.Mutex 49 subs []chan types.AttachmentUploadResult 50 res *types.AttachmentUploadResult 51 } 52 53 var _ types.AttachmentUploaderResultCb = (*uploaderResult)(nil) 54 55 func newUploaderResult() *uploaderResult { 56 return &uploaderResult{} 57 } 58 59 func (r *uploaderResult) Wait() (ch chan types.AttachmentUploadResult) { 60 r.Lock() 61 defer r.Unlock() 62 ch = make(chan types.AttachmentUploadResult, 1) 63 if r.res != nil { 64 // If we already have received a result and something waits, just return it right away 65 ch <- *r.res 66 return ch 67 } 68 r.subs = append(r.subs, ch) 69 return ch 70 } 71 72 func (r *uploaderResult) trigger(res types.AttachmentUploadResult) { 73 r.Lock() 74 defer r.Unlock() 75 r.res = &res 76 for _, sub := range r.subs { 77 sub <- res 78 } 79 } 80 81 type uploaderTaskStorage struct { 82 globals.Contextified 83 utils.DebugLabeler 84 } 85 86 func newUploaderTaskStorage(g *globals.Context) *uploaderTaskStorage { 87 return &uploaderTaskStorage{ 88 Contextified: globals.NewContextified(g), 89 DebugLabeler: utils.NewDebugLabeler(g.ExternalG(), "uploaderTaskStorage", false), 90 } 91 } 92 93 func (u *uploaderTaskStorage) getDir() string { 94 return filepath.Join(u.G().GetEnv().GetSharedDataDir(), "uploadertasks") 95 } 96 97 func (u *uploaderTaskStorage) taskOutboxIDPath(outboxID chat1.OutboxID) string { 98 return filepath.Join(u.getDir(), fmt.Sprintf("task_%s", outboxID.String())) 99 } 100 101 func (u *uploaderTaskStorage) statusOutboxIDPath(outboxID chat1.OutboxID) string { 102 return filepath.Join(u.getDir(), fmt.Sprintf("status_%s", outboxID.String())) 103 } 104 105 func (u *uploaderTaskStorage) file(outboxID chat1.OutboxID, getPath func(chat1.OutboxID) string) (*encrypteddb.EncryptedFile, error) { 106 dir := u.getDir() 107 if err := os.MkdirAll(dir, os.ModePerm); err != nil { 108 return nil, err 109 } 110 return encrypteddb.NewFile(u.G().ExternalG(), getPath(outboxID), 111 func(ctx context.Context) ([32]byte, error) { 112 return storage.GetSecretBoxKey(ctx, u.G().ExternalG()) 113 }), nil 114 } 115 116 func (u *uploaderTaskStorage) saveTask(ctx context.Context, task uploaderTask) error { 117 tf, err := u.file(task.OutboxID, u.taskOutboxIDPath) 118 if err != nil { 119 return err 120 } 121 return tf.Put(ctx, task) 122 } 123 124 func (u *uploaderTaskStorage) getTask(ctx context.Context, outboxID chat1.OutboxID) (res uploaderTask, err error) { 125 tf, err := u.file(outboxID, u.taskOutboxIDPath) 126 if err != nil { 127 return res, err 128 } 129 if err := tf.Get(ctx, &res); err != nil { 130 return res, err 131 } 132 return res, nil 133 } 134 135 func (u *uploaderTaskStorage) completeTask(ctx context.Context, outboxID chat1.OutboxID) { 136 if err := os.Remove(u.taskOutboxIDPath(outboxID)); err != nil { 137 u.Debug(ctx, "completeTask: failed to remove task file: outboxID: %s err: %s", outboxID, err) 138 } 139 if err := os.Remove(u.statusOutboxIDPath(outboxID)); err != nil { 140 u.Debug(ctx, "completeTask: failed to remove status file: outboxID: %s err: %s", outboxID, err) 141 } 142 } 143 144 func (u *uploaderTaskStorage) setStatus(ctx context.Context, outboxID chat1.OutboxID, status uploaderStatus) error { 145 sf, err := u.file(outboxID, u.statusOutboxIDPath) 146 if err != nil { 147 return err 148 } 149 return sf.Put(ctx, status) 150 } 151 152 func (u *uploaderTaskStorage) getStatus(ctx context.Context, outboxID chat1.OutboxID) (res uploaderStatus, err error) { 153 sf, err := u.file(outboxID, u.statusOutboxIDPath) 154 if err != nil { 155 return res, err 156 } 157 if err := sf.Get(ctx, &res); err != nil { 158 return res, err 159 } 160 return res, nil 161 } 162 163 type activeUpload struct { 164 uploadCtx context.Context 165 uploadCancelFn context.CancelFunc 166 uploadResult *uploaderResult 167 } 168 169 type Uploader struct { 170 globals.Contextified 171 utils.DebugLabeler 172 sync.Mutex 173 174 store Store 175 taskStorage *uploaderTaskStorage 176 ri func() chat1.RemoteInterface 177 s3signer s3.Signer 178 uploads map[string]*activeUpload 179 previewsLRU, fullsLRU *disklru.DiskLRU 180 versionUploaderTemps int 181 182 // testing 183 tempDir string 184 } 185 186 var _ types.AttachmentUploader = (*Uploader)(nil) 187 188 func NewUploader(g *globals.Context, store Store, s3signer s3.Signer, 189 ri func() chat1.RemoteInterface, size int) *Uploader { 190 u := &Uploader{ 191 Contextified: globals.NewContextified(g), 192 DebugLabeler: utils.NewDebugLabeler(g.ExternalG(), "Attachments.Uploader", false), 193 store: store, 194 ri: ri, 195 s3signer: s3signer, 196 uploads: make(map[string]*activeUpload), 197 taskStorage: newUploaderTaskStorage(g), 198 previewsLRU: disklru.NewDiskLRU(uploadedPreviewsDir, 2, size), 199 fullsLRU: disklru.NewDiskLRU(uploadedFullsDir, 2, size), 200 versionUploaderTemps: 1, 201 } 202 203 // make sure local state is clean 204 mctx := libkb.NewMetaContextTODO(g.ExternalG()) 205 go u.clearOldUploaderTempDirs(context.Background(), 8*time.Second) 206 go disklru.CleanOutOfSyncWithDelay(mctx, u.previewsLRU, u.getPreviewsDir(), 10*time.Second) 207 go disklru.CleanOutOfSyncWithDelay(mctx, u.fullsLRU, u.getFullsDir(), 10*time.Second) 208 return u 209 } 210 211 func (u *Uploader) SetPreviewTempDir(dir string) { 212 u.Lock() 213 defer u.Unlock() 214 u.tempDir = dir 215 } 216 217 func (u *Uploader) Complete(ctx context.Context, outboxID chat1.OutboxID) { 218 defer u.Trace(ctx, nil, "Complete(%s)", outboxID)() 219 status, err := u.getStatus(ctx, outboxID) 220 if err != nil { 221 u.Debug(ctx, "Complete: failed to get outboxID: %s", err) 222 return 223 } 224 if status.Status == types.AttachmentUploaderTaskStatusUploading { 225 u.Debug(ctx, "Complete: called on uploading attachment, ignoring: outboxID: %s", outboxID) 226 return 227 } 228 u.taskStorage.completeTask(ctx, outboxID) 229 NewPendingPreviews(u.G()).Remove(ctx, outboxID) 230 // just always attempt to remove the upload temp dir for this outbox ID, even if it might not be there 231 u.clearTempDirFromOutboxID(ctx, outboxID) 232 } 233 234 func (u *Uploader) clearExpoAudioPaths(ctx context.Context) { 235 if u.G().IsMobileAppType() { 236 var subDir string 237 if libkb.IsAndroid() { 238 subDir = "../../../cache/Audio" 239 } else { 240 subDir = "../AV" 241 } 242 dir := filepath.Join(u.G().GetCacheDir(), subDir) 243 u.Debug(ctx, "clearExpoAudioPaths: clearing current dir: %s", dir) 244 os.RemoveAll(dir) 245 } 246 } 247 248 func (u *Uploader) clearOldUploaderTempDirs(ctx context.Context, delay time.Duration) { 249 u.Debug(ctx, "clearOldUploaderTempDirs: cleaning in %v", delay) 250 select { 251 case <-ctx.Done(): 252 u.Debug(ctx, "clearOldUploaderTempDirs: context canceled, bailing") 253 return 254 case <-time.After(delay): 255 } 256 257 defer u.Trace(ctx, nil, "clearOldUploaderTempDirs")() 258 for i := 0; i < u.versionUploaderTemps; i++ { 259 dir := u.getUploadTempBaseDir(i) 260 u.Debug(ctx, "clearOldUploaderTempDirs: cleaning: %s", dir) 261 os.RemoveAll(dir) 262 } 263 if u.G().IsMobileAppType() { 264 u.clearExpoAudioPaths(ctx) 265 } else { 266 dir := u.getUploadTempBaseDir(u.versionUploaderTemps) 267 u.Debug(ctx, "clearOldUploaderTempDirs: clearing current dir: %s", dir) 268 os.RemoveAll(dir) 269 } 270 } 271 272 func (u *Uploader) clearTempDirFromOutboxID(ctx context.Context, outboxID chat1.OutboxID) { 273 dir := u.getUploadTempDir(u.versionUploaderTemps, outboxID) 274 u.Debug(ctx, "clearTempDirFromOutboxID: clearing: %s", dir) 275 os.RemoveAll(dir) 276 277 u.clearExpoAudioPaths(ctx) 278 } 279 280 func (u *Uploader) Retry(ctx context.Context, outboxID chat1.OutboxID) (res types.AttachmentUploaderResultCb, err error) { 281 defer u.Trace(ctx, &err, "Retry(%s)", outboxID)() 282 ustatus, err := u.getStatus(ctx, outboxID) 283 if err != nil { 284 return nil, err 285 } 286 switch ustatus.Status { 287 case types.AttachmentUploaderTaskStatusUploading, types.AttachmentUploaderTaskStatusFailed: 288 task, err := u.getTask(ctx, outboxID) 289 if err != nil { 290 return nil, err 291 } 292 return u.upload(ctx, task.UID, task.ConvID, task.OutboxID, task.Title, task.Filename, task.Metadata, 293 task.CallerPreview) 294 case types.AttachmentUploaderTaskStatusSuccess: 295 ur := newUploaderResult() 296 ur.trigger(ustatus.Result) 297 return ur, nil 298 } 299 return nil, fmt.Errorf("unknown retry status: %v", ustatus.Status) 300 } 301 302 func (u *Uploader) Cancel(ctx context.Context, outboxID chat1.OutboxID) (err error) { 303 defer u.Trace(ctx, &err, "Cancel(%s)", outboxID)() 304 // check if we are actively uploading the outbox ID and cancel it 305 u.Lock() 306 var ch chan types.AttachmentUploadResult 307 existing := u.uploads[outboxID.String()] 308 if existing != nil { 309 existing.uploadCancelFn() 310 ch = existing.uploadResult.Wait() 311 } 312 u.Unlock() 313 314 // Wait for the uploader to cancel 315 if ch != nil { 316 <-ch 317 } 318 319 // Take the whole record out of commission 320 u.Complete(ctx, outboxID) 321 return nil 322 } 323 324 func (u *Uploader) Status(ctx context.Context, outboxID chat1.OutboxID) (status types.AttachmentUploaderTaskStatus, res types.AttachmentUploadResult, err error) { 325 defer u.Trace(ctx, &err, "Status(%s)", outboxID)() 326 ustatus, err := u.getStatus(ctx, outboxID) 327 if err != nil { 328 return status, res, err 329 } 330 return ustatus.Status, ustatus.Result, nil 331 } 332 333 func (u *Uploader) getStatus(ctx context.Context, outboxID chat1.OutboxID) (res uploaderStatus, err error) { 334 return u.taskStorage.getStatus(ctx, outboxID) 335 } 336 337 func (u *Uploader) setStatus(ctx context.Context, outboxID chat1.OutboxID, status uploaderStatus) error { 338 return u.taskStorage.setStatus(ctx, outboxID, status) 339 } 340 341 func (u *Uploader) getTask(ctx context.Context, outboxID chat1.OutboxID) (uploaderTask, error) { 342 return u.taskStorage.getTask(ctx, outboxID) 343 } 344 345 func (u *Uploader) saveTask(ctx context.Context, uid gregor1.UID, convID chat1.ConversationID, 346 outboxID chat1.OutboxID, title, filename string, metadata []byte, callerPreview *chat1.MakePreviewRes) error { 347 task := uploaderTask{ 348 UID: uid, 349 OutboxID: outboxID, 350 ConvID: convID, 351 Title: title, 352 Filename: filename, 353 Metadata: metadata, 354 CallerPreview: callerPreview, 355 } 356 if err := u.taskStorage.saveTask(ctx, task); err != nil { 357 return err 358 } 359 return nil 360 } 361 362 func (u *Uploader) Register(ctx context.Context, uid gregor1.UID, convID chat1.ConversationID, 363 outboxID chat1.OutboxID, title, filename string, metadata []byte, callerPreview *chat1.MakePreviewRes) (res types.AttachmentUploaderResultCb, err error) { 364 defer u.Trace(ctx, &err, "Register(%s)", outboxID)() 365 // Write down the task information 366 if err := u.saveTask(ctx, uid, convID, outboxID, title, filename, metadata, callerPreview); err != nil { 367 return nil, err 368 } 369 var ustatus uploaderStatus 370 ustatus.Status = types.AttachmentUploaderTaskStatusUploading 371 if err := u.setStatus(ctx, outboxID, ustatus); err != nil { 372 return nil, err 373 } 374 // Start upload 375 return u.upload(ctx, uid, convID, outboxID, title, filename, metadata, callerPreview) 376 } 377 378 func (u *Uploader) checkAndSetUploading(uploadCtx context.Context, outboxID chat1.OutboxID, 379 uploadCancelFn context.CancelFunc) (upload *activeUpload, inprogress bool) { 380 u.Lock() 381 defer u.Unlock() 382 if upload = u.uploads[outboxID.String()]; upload != nil { 383 return upload, true 384 } 385 upload = &activeUpload{ 386 uploadCtx: uploadCtx, 387 uploadCancelFn: uploadCancelFn, 388 uploadResult: newUploaderResult(), 389 } 390 u.uploads[outboxID.String()] = upload 391 return upload, false 392 } 393 394 func (u *Uploader) doneUploading(outboxID chat1.OutboxID) { 395 u.Lock() 396 defer u.Unlock() 397 if existing := u.uploads[outboxID.String()]; existing != nil { 398 existing.uploadCancelFn() 399 } 400 delete(u.uploads, outboxID.String()) 401 } 402 403 func (u *Uploader) getBaseDir() string { 404 u.Lock() 405 defer u.Unlock() 406 baseDir := u.G().GetCacheDir() 407 if u.tempDir != "" { 408 baseDir = u.tempDir 409 } 410 return baseDir 411 } 412 413 // normalizeFilenameFromCache substitutes the existing cache dir value into the 414 // file path since it's possible for the path to the cache dir to change, 415 // especially on mobile. 416 func (u *Uploader) normalizeFilenameFromCache(dir, file string) string { 417 file = filepath.Base(file) 418 return filepath.Join(dir, file) 419 } 420 421 func (u *Uploader) uploadFile(ctx context.Context, diskLRU *disklru.DiskLRU, dirname, prefix string) (f *os.File, err error) { 422 baseDir := u.getBaseDir() 423 dir := filepath.Join(baseDir, dirname) 424 if err := os.MkdirAll(dir, os.ModePerm); err != nil { 425 return nil, err 426 } 427 f, err = os.CreateTemp(dir, prefix) 428 if err != nil { 429 return nil, err 430 } 431 432 // Add an entry to the disk LRU mapping with the tmpfilename to limit the 433 // number of resources on disk. If we evict something we remove the 434 // remnants. 435 evicted, err := diskLRU.Put(ctx, u.G(), f.Name(), f.Name()) 436 if err != nil { 437 return nil, err 438 } 439 if evicted != nil { 440 path := u.normalizeFilenameFromCache(dir, evicted.Value.(string)) 441 if oerr := os.Remove(path); oerr != nil { 442 u.Debug(ctx, "failed to remove file at %s, %v", path, oerr) 443 } 444 } 445 return f, nil 446 } 447 448 func (u *Uploader) uploadPreviewFile(ctx context.Context) (f *os.File, err error) { 449 return u.uploadFile(ctx, u.previewsLRU, uploadedPreviewsDir, "up") 450 } 451 452 func (u *Uploader) uploadFullFile(ctx context.Context, md chat1.AssetMetadata) (f *os.File, err error) { 453 // make sure we want to stash this full asset in our local cache 454 typ, err := md.AssetType() 455 if err != nil { 456 return nil, err 457 } 458 switch typ { 459 case chat1.AssetMetadataType_IMAGE: 460 // we will stash these guys 461 default: 462 return nil, fmt.Errorf("not storing full of type: %v", typ) 463 } 464 return u.uploadFile(ctx, u.fullsLRU, uploadedFullsDir, "fl") 465 } 466 467 func (u *Uploader) upload(ctx context.Context, uid gregor1.UID, convID chat1.ConversationID, 468 outboxID chat1.OutboxID, title, filename string, metadata []byte, callerPreview *chat1.MakePreviewRes) (res types.AttachmentUploaderResultCb, err error) { 469 470 // Create the errgroup first so we can register the context in the upload map 471 var g *errgroup.Group 472 var cancelFn context.CancelFunc 473 bgctx := libkb.CopyTagsToBackground(ctx) 474 g, bgctx = errgroup.WithContext(bgctx) 475 if os.Getenv("CHAT_S3_FAKE") == "1" { 476 bgctx = s3.NewFakeS3Context(bgctx) 477 } 478 bgctx, cancelFn = context.WithCancel(bgctx) 479 480 // Check to see if we are already uploading this message and set upload status if not 481 upload, inprogress := u.checkAndSetUploading(bgctx, outboxID, cancelFn) 482 if inprogress { 483 u.Debug(ctx, "upload: already uploading: %s, returning early", outboxID) 484 return upload.uploadResult, nil 485 } 486 defer func() { 487 if err != nil { 488 // we only get an error back here if we didn't actually start upload, so stop it here now 489 u.doneUploading(outboxID) 490 } 491 }() 492 493 // Stat the file to get size 494 finfo, err := StatOSOrKbfsFile(ctx, u.G().GlobalContext, filename) 495 if err != nil { 496 return res, err 497 } 498 src, err := NewReadCloseResetter(bgctx, u.G().GlobalContext, filename) 499 if err != nil { 500 return res, err 501 } 502 503 deferToBackgroundRoutine := false 504 defer func() { 505 if !deferToBackgroundRoutine { 506 src.Close() 507 } 508 }() 509 510 progress := func(bytesComplete, bytesTotal int64) { 511 u.G().ActivityNotifier.AttachmentUploadProgress(ctx, uid, convID, outboxID, bytesComplete, bytesTotal) 512 } 513 514 // preprocess asset (get content type, create preview if possible, convert from heic to jpeg) 515 var pre Preprocess 516 var ures types.AttachmentUploadResult 517 ures.Metadata = metadata 518 pp := NewPendingPreviews(u.G()) 519 fileSize := finfo.Size() 520 if pre, err = pp.Get(ctx, outboxID); err != nil { 521 u.Debug(ctx, "upload: no pending preview, generating one: %s", err) 522 if pre, err = PreprocessAsset(ctx, u.G(), u.DebugLabeler, src, filename, u.G().NativeVideoHelper, 523 callerPreview); err != nil { 524 u.Debug(ctx, "upload: failed to preprocess: %s", err) 525 return res, err 526 } 527 if pre.Preview != nil { 528 u.Debug(ctx, "upload: created preview in preprocess") 529 // Store the preview in pending storage 530 if err = pp.Put(ctx, outboxID, pre); err != nil { 531 return res, err 532 } 533 } 534 } 535 536 filename = pre.Filename 537 // Use our converted input, if available 538 if pre.SrcDat != nil { 539 fileSize = int64(len(pre.SrcDat)) 540 src.Close() 541 src = NewBufReadResetter(pre.SrcDat) 542 } 543 544 var s3params chat1.S3Params 545 paramsCh := make(chan struct{}) 546 g.Go(func() (err error) { 547 u.Debug(bgctx, "upload: fetching s3 params") 548 u.G().ActivityNotifier.AttachmentUploadStart(bgctx, uid, convID, outboxID) 549 if s3params, err = u.ri().GetS3Params(bgctx, chat1.GetS3ParamsArg{ 550 ConversationID: convID, 551 TempCreds: true, 552 }); err != nil { 553 return err 554 } 555 close(paramsCh) 556 return nil 557 }) 558 // upload attachment and (optional) preview concurrently 559 g.Go(func() (err error) { 560 select { 561 case <-paramsCh: 562 case <-bgctx.Done(): 563 return bgctx.Err() 564 } 565 566 // set up file to write out encrypted preview to 567 var encryptedOut io.Writer 568 uf, err := u.uploadFullFile(ctx, pre.BaseMetadata()) 569 if err != nil { 570 u.Debug(bgctx, "upload: failed to create uploaded full file: %s", err) 571 encryptedOut = io.Discard 572 uf = nil 573 } else { 574 defer uf.Close() 575 encryptedOut = uf 576 } 577 578 u.Debug(bgctx, "upload: uploading assets") 579 task := UploadTask{ 580 S3Params: s3params, 581 Filename: filename, 582 FileSize: fileSize, 583 Plaintext: src, 584 S3Signer: u.s3signer, 585 ConversationID: convID, 586 UserID: uid, 587 OutboxID: outboxID, 588 Preview: false, 589 Progress: progress, 590 } 591 ures.Object, err = u.store.UploadAsset(bgctx, &task, encryptedOut) 592 if err != nil { 593 u.Debug(bgctx, "upload: error uploading primary asset to s3: %s", err) 594 } else { 595 ures.Object.Title = title 596 ures.Object.MimeType = pre.ContentType 597 ures.Object.Metadata = pre.BaseMetadata() 598 if uf != nil { 599 if err := u.G().AttachmentURLSrv.GetAttachmentFetcher().PutUploadedAsset(ctx, 600 uf.Name(), ures.Object); err != nil { 601 u.Debug(bgctx, "upload: failed to put uploaded asset into fetcher: %s", err) 602 } 603 } 604 } 605 u.Debug(bgctx, "upload: asset upload complete") 606 return err 607 }) 608 if pre.Preview != nil { 609 g.Go(func() error { 610 select { 611 case <-paramsCh: 612 case <-bgctx.Done(): 613 return bgctx.Err() 614 } 615 616 // check to make sure this isn't an emoji conv, and if so just abort 617 conv, err := utils.GetUnverifiedConv(bgctx, u.G(), uid, convID, types.InboxSourceDataSourceAll) 618 if err != nil { 619 return err 620 } 621 switch conv.GetTopicType() { 622 case chat1.TopicType_EMOJI, chat1.TopicType_EMOJICROSS: 623 u.Debug(bgctx, "upload: skipping preview upload in emoji conv") 624 return nil 625 default: 626 } 627 628 // copy the params so as not to mess with the main params above 629 previewParams := s3params 630 631 // set up file to write out encrypted preview to 632 var encryptedOut io.Writer 633 up, err := u.uploadPreviewFile(ctx) 634 if err != nil { 635 u.Debug(bgctx, "upload: failed to create uploaded preview file: %s", err) 636 encryptedOut = io.Discard 637 up = nil 638 } else { 639 defer up.Close() 640 encryptedOut = up 641 } 642 643 // add preview suffix to object key (P in hex) 644 // the s3path in gregor is expecting hex here 645 previewParams.ObjectKey += "50" 646 task := UploadTask{ 647 S3Params: previewParams, 648 Filename: filename, 649 FileSize: int64(len(pre.Preview)), 650 Plaintext: NewBufReadResetter(pre.Preview), 651 S3Signer: u.s3signer, 652 ConversationID: convID, 653 UserID: uid, 654 OutboxID: outboxID, 655 Preview: true, 656 } 657 preview, err := u.store.UploadAsset(bgctx, &task, encryptedOut) 658 if err == nil { 659 ures.Preview = &preview 660 ures.Preview.MimeType = pre.PreviewContentType 661 ures.Preview.Metadata = pre.PreviewMetadata() 662 ures.Preview.Tag = chat1.AssetTag_PRIMARY 663 if up != nil { 664 if err := u.G().AttachmentURLSrv.GetAttachmentFetcher().PutUploadedAsset(ctx, 665 up.Name(), preview); err != nil { 666 u.Debug(bgctx, "upload: failed to put uploaded preview asset into fetcher: %s", err) 667 } 668 } 669 } else { 670 u.Debug(bgctx, "upload: error uploading preview asset to s3: %s", err) 671 } 672 u.Debug(bgctx, "upload: preview upload complete") 673 return err 674 }) 675 } 676 677 deferToBackgroundRoutine = true 678 go func() { 679 defer src.Close() 680 var errStr string 681 status := types.AttachmentUploaderTaskStatusSuccess 682 if err := g.Wait(); err != nil { 683 status = types.AttachmentUploaderTaskStatusFailed 684 ures.Error = new(string) 685 *ures.Error = err.Error() 686 errStr = err.Error() 687 } 688 if err := u.setStatus(bgctx, outboxID, uploaderStatus{ 689 Status: status, 690 Result: ures, 691 }); err != nil { 692 u.Debug(bgctx, "failed to set status on upload success: %s", err) 693 } 694 u.Debug(bgctx, "upload: upload complete: status: %v err: %s", status, errStr) 695 // Ping Deliverer to notify that some of the message in the outbox might be read to send 696 u.G().MessageDeliverer.ForceDeliverLoop(bgctx) 697 upload.uploadResult.trigger(ures) 698 u.doneUploading(outboxID) 699 }() 700 return upload.uploadResult, nil 701 } 702 703 func (u *Uploader) getUploadTempBaseDir(version int) string { 704 base := filepath.Join(u.G().GetSharedCacheDir(), uploadedTempsDir) 705 // version 0 didn't have the naming scheme ready, so special case it 706 if version == 0 { 707 return base 708 } 709 return fmt.Sprintf("%s_v%d", base, version) 710 } 711 712 func (u *Uploader) getUploadTempDir(version int, outboxID chat1.OutboxID) string { 713 return filepath.Join(u.getUploadTempBaseDir(version), outboxID.String()) 714 } 715 716 func (u *Uploader) GetUploadTempFile(ctx context.Context, outboxID chat1.OutboxID, filename string) (string, error) { 717 dir := u.getUploadTempDir(u.versionUploaderTemps, outboxID) 718 if err := os.MkdirAll(dir, os.ModePerm); err != nil { 719 return "", err 720 } 721 return filepath.Join(dir, filepath.Base(filename)), nil 722 } 723 724 func (u *Uploader) GetUploadTempSink(ctx context.Context, filename string) (*os.File, chat1.OutboxID, error) { 725 obid, err := storage.NewOutboxID() 726 if err != nil { 727 return nil, nil, err 728 } 729 filename, err = u.GetUploadTempFile(ctx, obid, filename) 730 if err != nil { 731 return nil, nil, err 732 } 733 file, err := os.Create(filename) 734 if err != nil { 735 return nil, nil, err 736 } 737 return file, obid, nil 738 } 739 740 func (u *Uploader) CancelUploadTempFile(ctx context.Context, outboxID chat1.OutboxID) error { 741 u.clearTempDirFromOutboxID(ctx, outboxID) 742 return nil 743 } 744 745 func (u *Uploader) getPreviewsDir() string { 746 return filepath.Join(u.getBaseDir(), uploadedPreviewsDir) 747 } 748 749 func (u *Uploader) getFullsDir() string { 750 return filepath.Join(u.getBaseDir(), uploadedFullsDir) 751 } 752 753 func (u *Uploader) OnDbNuke(mctx libkb.MetaContext) error { 754 if err := u.previewsLRU.CleanOutOfSync(mctx, u.getPreviewsDir()); err != nil { 755 u.Debug(mctx.Ctx(), "unable to run clean for uploadedPreviews: %v", err) 756 } 757 if err := u.fullsLRU.CleanOutOfSync(mctx, u.getFullsDir()); err != nil { 758 u.Debug(mctx.Ctx(), "unable to run clean for uploadedFulls: %v", err) 759 } 760 return nil 761 }