github.com/keybase/client/go@v0.0.0-20241007131713-f10651d043c8/chat/attachments/uploader.go (about)

     1  package attachments
     2  
     3  import (
     4  	"fmt"
     5  	"io"
     6  	"os"
     7  	"path/filepath"
     8  	"sync"
     9  	"time"
    10  
    11  	disklru "github.com/keybase/client/go/lru"
    12  
    13  	"github.com/keybase/client/go/encrypteddb"
    14  
    15  	"github.com/keybase/client/go/chat/globals"
    16  	"github.com/keybase/client/go/chat/s3"
    17  	"github.com/keybase/client/go/chat/storage"
    18  	"github.com/keybase/client/go/chat/types"
    19  	"github.com/keybase/client/go/chat/utils"
    20  	"github.com/keybase/client/go/libkb"
    21  	"github.com/keybase/client/go/protocol/chat1"
    22  	"github.com/keybase/client/go/protocol/gregor1"
    23  	"golang.org/x/net/context"
    24  	"golang.org/x/sync/errgroup"
    25  )
    26  
    27  const (
    28  	uploadedPreviewsDir = "uploadedpreviews"
    29  	uploadedFullsDir    = "uploadedfulls"
    30  	uploadedTempsDir    = "uploadtemps"
    31  )
    32  
    33  type uploaderTask struct {
    34  	UID             gregor1.UID
    35  	OutboxID        chat1.OutboxID
    36  	ConvID          chat1.ConversationID
    37  	Title, Filename string
    38  	Metadata        []byte
    39  	CallerPreview   *chat1.MakePreviewRes
    40  }
    41  
    42  type uploaderStatus struct {
    43  	Status types.AttachmentUploaderTaskStatus
    44  	Result types.AttachmentUploadResult
    45  }
    46  
    47  type uploaderResult struct {
    48  	sync.Mutex
    49  	subs []chan types.AttachmentUploadResult
    50  	res  *types.AttachmentUploadResult
    51  }
    52  
    53  var _ types.AttachmentUploaderResultCb = (*uploaderResult)(nil)
    54  
    55  func newUploaderResult() *uploaderResult {
    56  	return &uploaderResult{}
    57  }
    58  
    59  func (r *uploaderResult) Wait() (ch chan types.AttachmentUploadResult) {
    60  	r.Lock()
    61  	defer r.Unlock()
    62  	ch = make(chan types.AttachmentUploadResult, 1)
    63  	if r.res != nil {
    64  		// If we already have received a result and something waits, just return it right away
    65  		ch <- *r.res
    66  		return ch
    67  	}
    68  	r.subs = append(r.subs, ch)
    69  	return ch
    70  }
    71  
    72  func (r *uploaderResult) trigger(res types.AttachmentUploadResult) {
    73  	r.Lock()
    74  	defer r.Unlock()
    75  	r.res = &res
    76  	for _, sub := range r.subs {
    77  		sub <- res
    78  	}
    79  }
    80  
    81  type uploaderTaskStorage struct {
    82  	globals.Contextified
    83  	utils.DebugLabeler
    84  }
    85  
    86  func newUploaderTaskStorage(g *globals.Context) *uploaderTaskStorage {
    87  	return &uploaderTaskStorage{
    88  		Contextified: globals.NewContextified(g),
    89  		DebugLabeler: utils.NewDebugLabeler(g.ExternalG(), "uploaderTaskStorage", false),
    90  	}
    91  }
    92  
    93  func (u *uploaderTaskStorage) getDir() string {
    94  	return filepath.Join(u.G().GetEnv().GetSharedDataDir(), "uploadertasks")
    95  }
    96  
    97  func (u *uploaderTaskStorage) taskOutboxIDPath(outboxID chat1.OutboxID) string {
    98  	return filepath.Join(u.getDir(), fmt.Sprintf("task_%s", outboxID.String()))
    99  }
   100  
   101  func (u *uploaderTaskStorage) statusOutboxIDPath(outboxID chat1.OutboxID) string {
   102  	return filepath.Join(u.getDir(), fmt.Sprintf("status_%s", outboxID.String()))
   103  }
   104  
   105  func (u *uploaderTaskStorage) file(outboxID chat1.OutboxID, getPath func(chat1.OutboxID) string) (*encrypteddb.EncryptedFile, error) {
   106  	dir := u.getDir()
   107  	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
   108  		return nil, err
   109  	}
   110  	return encrypteddb.NewFile(u.G().ExternalG(), getPath(outboxID),
   111  		func(ctx context.Context) ([32]byte, error) {
   112  			return storage.GetSecretBoxKey(ctx, u.G().ExternalG())
   113  		}), nil
   114  }
   115  
   116  func (u *uploaderTaskStorage) saveTask(ctx context.Context, task uploaderTask) error {
   117  	tf, err := u.file(task.OutboxID, u.taskOutboxIDPath)
   118  	if err != nil {
   119  		return err
   120  	}
   121  	return tf.Put(ctx, task)
   122  }
   123  
   124  func (u *uploaderTaskStorage) getTask(ctx context.Context, outboxID chat1.OutboxID) (res uploaderTask, err error) {
   125  	tf, err := u.file(outboxID, u.taskOutboxIDPath)
   126  	if err != nil {
   127  		return res, err
   128  	}
   129  	if err := tf.Get(ctx, &res); err != nil {
   130  		return res, err
   131  	}
   132  	return res, nil
   133  }
   134  
   135  func (u *uploaderTaskStorage) completeTask(ctx context.Context, outboxID chat1.OutboxID) {
   136  	if err := os.Remove(u.taskOutboxIDPath(outboxID)); err != nil {
   137  		u.Debug(ctx, "completeTask: failed to remove task file: outboxID: %s err: %s", outboxID, err)
   138  	}
   139  	if err := os.Remove(u.statusOutboxIDPath(outboxID)); err != nil {
   140  		u.Debug(ctx, "completeTask: failed to remove status file: outboxID: %s err: %s", outboxID, err)
   141  	}
   142  }
   143  
   144  func (u *uploaderTaskStorage) setStatus(ctx context.Context, outboxID chat1.OutboxID, status uploaderStatus) error {
   145  	sf, err := u.file(outboxID, u.statusOutboxIDPath)
   146  	if err != nil {
   147  		return err
   148  	}
   149  	return sf.Put(ctx, status)
   150  }
   151  
   152  func (u *uploaderTaskStorage) getStatus(ctx context.Context, outboxID chat1.OutboxID) (res uploaderStatus, err error) {
   153  	sf, err := u.file(outboxID, u.statusOutboxIDPath)
   154  	if err != nil {
   155  		return res, err
   156  	}
   157  	if err := sf.Get(ctx, &res); err != nil {
   158  		return res, err
   159  	}
   160  	return res, nil
   161  }
   162  
   163  type activeUpload struct {
   164  	uploadCtx      context.Context
   165  	uploadCancelFn context.CancelFunc
   166  	uploadResult   *uploaderResult
   167  }
   168  
   169  type Uploader struct {
   170  	globals.Contextified
   171  	utils.DebugLabeler
   172  	sync.Mutex
   173  
   174  	store                 Store
   175  	taskStorage           *uploaderTaskStorage
   176  	ri                    func() chat1.RemoteInterface
   177  	s3signer              s3.Signer
   178  	uploads               map[string]*activeUpload
   179  	previewsLRU, fullsLRU *disklru.DiskLRU
   180  	versionUploaderTemps  int
   181  
   182  	// testing
   183  	tempDir string
   184  }
   185  
   186  var _ types.AttachmentUploader = (*Uploader)(nil)
   187  
   188  func NewUploader(g *globals.Context, store Store, s3signer s3.Signer,
   189  	ri func() chat1.RemoteInterface, size int) *Uploader {
   190  	u := &Uploader{
   191  		Contextified:         globals.NewContextified(g),
   192  		DebugLabeler:         utils.NewDebugLabeler(g.ExternalG(), "Attachments.Uploader", false),
   193  		store:                store,
   194  		ri:                   ri,
   195  		s3signer:             s3signer,
   196  		uploads:              make(map[string]*activeUpload),
   197  		taskStorage:          newUploaderTaskStorage(g),
   198  		previewsLRU:          disklru.NewDiskLRU(uploadedPreviewsDir, 2, size),
   199  		fullsLRU:             disklru.NewDiskLRU(uploadedFullsDir, 2, size),
   200  		versionUploaderTemps: 1,
   201  	}
   202  
   203  	// make sure local state is clean
   204  	mctx := libkb.NewMetaContextTODO(g.ExternalG())
   205  	go u.clearOldUploaderTempDirs(context.Background(), 8*time.Second)
   206  	go disklru.CleanOutOfSyncWithDelay(mctx, u.previewsLRU, u.getPreviewsDir(), 10*time.Second)
   207  	go disklru.CleanOutOfSyncWithDelay(mctx, u.fullsLRU, u.getFullsDir(), 10*time.Second)
   208  	return u
   209  }
   210  
   211  func (u *Uploader) SetPreviewTempDir(dir string) {
   212  	u.Lock()
   213  	defer u.Unlock()
   214  	u.tempDir = dir
   215  }
   216  
   217  func (u *Uploader) Complete(ctx context.Context, outboxID chat1.OutboxID) {
   218  	defer u.Trace(ctx, nil, "Complete(%s)", outboxID)()
   219  	status, err := u.getStatus(ctx, outboxID)
   220  	if err != nil {
   221  		u.Debug(ctx, "Complete: failed to get outboxID: %s", err)
   222  		return
   223  	}
   224  	if status.Status == types.AttachmentUploaderTaskStatusUploading {
   225  		u.Debug(ctx, "Complete: called on uploading attachment, ignoring: outboxID: %s", outboxID)
   226  		return
   227  	}
   228  	u.taskStorage.completeTask(ctx, outboxID)
   229  	NewPendingPreviews(u.G()).Remove(ctx, outboxID)
   230  	// just always attempt to remove the upload temp dir for this outbox ID, even if it might not be there
   231  	u.clearTempDirFromOutboxID(ctx, outboxID)
   232  }
   233  
   234  func (u *Uploader) clearExpoAudioPaths(ctx context.Context) {
   235  	if u.G().IsMobileAppType() {
   236  		var subDir string
   237  		if libkb.IsAndroid() {
   238  			subDir = "../../../cache/Audio"
   239  		} else {
   240  			subDir = "../AV"
   241  		}
   242  		dir := filepath.Join(u.G().GetCacheDir(), subDir)
   243  		u.Debug(ctx, "clearExpoAudioPaths: clearing current dir: %s", dir)
   244  		os.RemoveAll(dir)
   245  	}
   246  }
   247  
   248  func (u *Uploader) clearOldUploaderTempDirs(ctx context.Context, delay time.Duration) {
   249  	u.Debug(ctx, "clearOldUploaderTempDirs: cleaning in %v", delay)
   250  	select {
   251  	case <-ctx.Done():
   252  		u.Debug(ctx, "clearOldUploaderTempDirs: context canceled, bailing")
   253  		return
   254  	case <-time.After(delay):
   255  	}
   256  
   257  	defer u.Trace(ctx, nil, "clearOldUploaderTempDirs")()
   258  	for i := 0; i < u.versionUploaderTemps; i++ {
   259  		dir := u.getUploadTempBaseDir(i)
   260  		u.Debug(ctx, "clearOldUploaderTempDirs: cleaning: %s", dir)
   261  		os.RemoveAll(dir)
   262  	}
   263  	if u.G().IsMobileAppType() {
   264  		u.clearExpoAudioPaths(ctx)
   265  	} else {
   266  		dir := u.getUploadTempBaseDir(u.versionUploaderTemps)
   267  		u.Debug(ctx, "clearOldUploaderTempDirs: clearing current dir: %s", dir)
   268  		os.RemoveAll(dir)
   269  	}
   270  }
   271  
   272  func (u *Uploader) clearTempDirFromOutboxID(ctx context.Context, outboxID chat1.OutboxID) {
   273  	dir := u.getUploadTempDir(u.versionUploaderTemps, outboxID)
   274  	u.Debug(ctx, "clearTempDirFromOutboxID: clearing: %s", dir)
   275  	os.RemoveAll(dir)
   276  
   277  	u.clearExpoAudioPaths(ctx)
   278  }
   279  
   280  func (u *Uploader) Retry(ctx context.Context, outboxID chat1.OutboxID) (res types.AttachmentUploaderResultCb, err error) {
   281  	defer u.Trace(ctx, &err, "Retry(%s)", outboxID)()
   282  	ustatus, err := u.getStatus(ctx, outboxID)
   283  	if err != nil {
   284  		return nil, err
   285  	}
   286  	switch ustatus.Status {
   287  	case types.AttachmentUploaderTaskStatusUploading, types.AttachmentUploaderTaskStatusFailed:
   288  		task, err := u.getTask(ctx, outboxID)
   289  		if err != nil {
   290  			return nil, err
   291  		}
   292  		return u.upload(ctx, task.UID, task.ConvID, task.OutboxID, task.Title, task.Filename, task.Metadata,
   293  			task.CallerPreview)
   294  	case types.AttachmentUploaderTaskStatusSuccess:
   295  		ur := newUploaderResult()
   296  		ur.trigger(ustatus.Result)
   297  		return ur, nil
   298  	}
   299  	return nil, fmt.Errorf("unknown retry status: %v", ustatus.Status)
   300  }
   301  
   302  func (u *Uploader) Cancel(ctx context.Context, outboxID chat1.OutboxID) (err error) {
   303  	defer u.Trace(ctx, &err, "Cancel(%s)", outboxID)()
   304  	// check if we are actively uploading the outbox ID and cancel it
   305  	u.Lock()
   306  	var ch chan types.AttachmentUploadResult
   307  	existing := u.uploads[outboxID.String()]
   308  	if existing != nil {
   309  		existing.uploadCancelFn()
   310  		ch = existing.uploadResult.Wait()
   311  	}
   312  	u.Unlock()
   313  
   314  	// Wait for the uploader to cancel
   315  	if ch != nil {
   316  		<-ch
   317  	}
   318  
   319  	// Take the whole record out of commission
   320  	u.Complete(ctx, outboxID)
   321  	return nil
   322  }
   323  
   324  func (u *Uploader) Status(ctx context.Context, outboxID chat1.OutboxID) (status types.AttachmentUploaderTaskStatus, res types.AttachmentUploadResult, err error) {
   325  	defer u.Trace(ctx, &err, "Status(%s)", outboxID)()
   326  	ustatus, err := u.getStatus(ctx, outboxID)
   327  	if err != nil {
   328  		return status, res, err
   329  	}
   330  	return ustatus.Status, ustatus.Result, nil
   331  }
   332  
   333  func (u *Uploader) getStatus(ctx context.Context, outboxID chat1.OutboxID) (res uploaderStatus, err error) {
   334  	return u.taskStorage.getStatus(ctx, outboxID)
   335  }
   336  
   337  func (u *Uploader) setStatus(ctx context.Context, outboxID chat1.OutboxID, status uploaderStatus) error {
   338  	return u.taskStorage.setStatus(ctx, outboxID, status)
   339  }
   340  
   341  func (u *Uploader) getTask(ctx context.Context, outboxID chat1.OutboxID) (uploaderTask, error) {
   342  	return u.taskStorage.getTask(ctx, outboxID)
   343  }
   344  
   345  func (u *Uploader) saveTask(ctx context.Context, uid gregor1.UID, convID chat1.ConversationID,
   346  	outboxID chat1.OutboxID, title, filename string, metadata []byte, callerPreview *chat1.MakePreviewRes) error {
   347  	task := uploaderTask{
   348  		UID:           uid,
   349  		OutboxID:      outboxID,
   350  		ConvID:        convID,
   351  		Title:         title,
   352  		Filename:      filename,
   353  		Metadata:      metadata,
   354  		CallerPreview: callerPreview,
   355  	}
   356  	if err := u.taskStorage.saveTask(ctx, task); err != nil {
   357  		return err
   358  	}
   359  	return nil
   360  }
   361  
   362  func (u *Uploader) Register(ctx context.Context, uid gregor1.UID, convID chat1.ConversationID,
   363  	outboxID chat1.OutboxID, title, filename string, metadata []byte, callerPreview *chat1.MakePreviewRes) (res types.AttachmentUploaderResultCb, err error) {
   364  	defer u.Trace(ctx, &err, "Register(%s)", outboxID)()
   365  	// Write down the task information
   366  	if err := u.saveTask(ctx, uid, convID, outboxID, title, filename, metadata, callerPreview); err != nil {
   367  		return nil, err
   368  	}
   369  	var ustatus uploaderStatus
   370  	ustatus.Status = types.AttachmentUploaderTaskStatusUploading
   371  	if err := u.setStatus(ctx, outboxID, ustatus); err != nil {
   372  		return nil, err
   373  	}
   374  	// Start upload
   375  	return u.upload(ctx, uid, convID, outboxID, title, filename, metadata, callerPreview)
   376  }
   377  
   378  func (u *Uploader) checkAndSetUploading(uploadCtx context.Context, outboxID chat1.OutboxID,
   379  	uploadCancelFn context.CancelFunc) (upload *activeUpload, inprogress bool) {
   380  	u.Lock()
   381  	defer u.Unlock()
   382  	if upload = u.uploads[outboxID.String()]; upload != nil {
   383  		return upload, true
   384  	}
   385  	upload = &activeUpload{
   386  		uploadCtx:      uploadCtx,
   387  		uploadCancelFn: uploadCancelFn,
   388  		uploadResult:   newUploaderResult(),
   389  	}
   390  	u.uploads[outboxID.String()] = upload
   391  	return upload, false
   392  }
   393  
   394  func (u *Uploader) doneUploading(outboxID chat1.OutboxID) {
   395  	u.Lock()
   396  	defer u.Unlock()
   397  	if existing := u.uploads[outboxID.String()]; existing != nil {
   398  		existing.uploadCancelFn()
   399  	}
   400  	delete(u.uploads, outboxID.String())
   401  }
   402  
   403  func (u *Uploader) getBaseDir() string {
   404  	u.Lock()
   405  	defer u.Unlock()
   406  	baseDir := u.G().GetCacheDir()
   407  	if u.tempDir != "" {
   408  		baseDir = u.tempDir
   409  	}
   410  	return baseDir
   411  }
   412  
   413  // normalizeFilenameFromCache substitutes the existing cache dir value into the
   414  // file path since it's possible for the path to the cache dir to change,
   415  // especially on mobile.
   416  func (u *Uploader) normalizeFilenameFromCache(dir, file string) string {
   417  	file = filepath.Base(file)
   418  	return filepath.Join(dir, file)
   419  }
   420  
   421  func (u *Uploader) uploadFile(ctx context.Context, diskLRU *disklru.DiskLRU, dirname, prefix string) (f *os.File, err error) {
   422  	baseDir := u.getBaseDir()
   423  	dir := filepath.Join(baseDir, dirname)
   424  	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
   425  		return nil, err
   426  	}
   427  	f, err = os.CreateTemp(dir, prefix)
   428  	if err != nil {
   429  		return nil, err
   430  	}
   431  
   432  	// Add an entry to the disk LRU mapping with the tmpfilename to limit the
   433  	// number of resources on disk. If we evict something we remove the
   434  	// remnants.
   435  	evicted, err := diskLRU.Put(ctx, u.G(), f.Name(), f.Name())
   436  	if err != nil {
   437  		return nil, err
   438  	}
   439  	if evicted != nil {
   440  		path := u.normalizeFilenameFromCache(dir, evicted.Value.(string))
   441  		if oerr := os.Remove(path); oerr != nil {
   442  			u.Debug(ctx, "failed to remove file at %s, %v", path, oerr)
   443  		}
   444  	}
   445  	return f, nil
   446  }
   447  
   448  func (u *Uploader) uploadPreviewFile(ctx context.Context) (f *os.File, err error) {
   449  	return u.uploadFile(ctx, u.previewsLRU, uploadedPreviewsDir, "up")
   450  }
   451  
   452  func (u *Uploader) uploadFullFile(ctx context.Context, md chat1.AssetMetadata) (f *os.File, err error) {
   453  	// make sure we want to stash this full asset in our local cache
   454  	typ, err := md.AssetType()
   455  	if err != nil {
   456  		return nil, err
   457  	}
   458  	switch typ {
   459  	case chat1.AssetMetadataType_IMAGE:
   460  		// we will stash these guys
   461  	default:
   462  		return nil, fmt.Errorf("not storing full of type: %v", typ)
   463  	}
   464  	return u.uploadFile(ctx, u.fullsLRU, uploadedFullsDir, "fl")
   465  }
   466  
   467  func (u *Uploader) upload(ctx context.Context, uid gregor1.UID, convID chat1.ConversationID,
   468  	outboxID chat1.OutboxID, title, filename string, metadata []byte, callerPreview *chat1.MakePreviewRes) (res types.AttachmentUploaderResultCb, err error) {
   469  
   470  	// Create the errgroup first so we can register the context in the upload map
   471  	var g *errgroup.Group
   472  	var cancelFn context.CancelFunc
   473  	bgctx := libkb.CopyTagsToBackground(ctx)
   474  	g, bgctx = errgroup.WithContext(bgctx)
   475  	if os.Getenv("CHAT_S3_FAKE") == "1" {
   476  		bgctx = s3.NewFakeS3Context(bgctx)
   477  	}
   478  	bgctx, cancelFn = context.WithCancel(bgctx)
   479  
   480  	// Check to see if we are already uploading this message and set upload status if not
   481  	upload, inprogress := u.checkAndSetUploading(bgctx, outboxID, cancelFn)
   482  	if inprogress {
   483  		u.Debug(ctx, "upload: already uploading: %s, returning early", outboxID)
   484  		return upload.uploadResult, nil
   485  	}
   486  	defer func() {
   487  		if err != nil {
   488  			// we only get an error back here if we didn't actually start upload, so stop it here now
   489  			u.doneUploading(outboxID)
   490  		}
   491  	}()
   492  
   493  	// Stat the file to get size
   494  	finfo, err := StatOSOrKbfsFile(ctx, u.G().GlobalContext, filename)
   495  	if err != nil {
   496  		return res, err
   497  	}
   498  	src, err := NewReadCloseResetter(bgctx, u.G().GlobalContext, filename)
   499  	if err != nil {
   500  		return res, err
   501  	}
   502  
   503  	deferToBackgroundRoutine := false
   504  	defer func() {
   505  		if !deferToBackgroundRoutine {
   506  			src.Close()
   507  		}
   508  	}()
   509  
   510  	progress := func(bytesComplete, bytesTotal int64) {
   511  		u.G().ActivityNotifier.AttachmentUploadProgress(ctx, uid, convID, outboxID, bytesComplete, bytesTotal)
   512  	}
   513  
   514  	// preprocess asset (get content type, create preview if possible, convert from heic to jpeg)
   515  	var pre Preprocess
   516  	var ures types.AttachmentUploadResult
   517  	ures.Metadata = metadata
   518  	pp := NewPendingPreviews(u.G())
   519  	fileSize := finfo.Size()
   520  	if pre, err = pp.Get(ctx, outboxID); err != nil {
   521  		u.Debug(ctx, "upload: no pending preview, generating one: %s", err)
   522  		if pre, err = PreprocessAsset(ctx, u.G(), u.DebugLabeler, src, filename, u.G().NativeVideoHelper,
   523  			callerPreview); err != nil {
   524  			u.Debug(ctx, "upload: failed to preprocess: %s", err)
   525  			return res, err
   526  		}
   527  		if pre.Preview != nil {
   528  			u.Debug(ctx, "upload: created preview in preprocess")
   529  			// Store the preview in pending storage
   530  			if err = pp.Put(ctx, outboxID, pre); err != nil {
   531  				return res, err
   532  			}
   533  		}
   534  	}
   535  
   536  	filename = pre.Filename
   537  	// Use our converted input, if available
   538  	if pre.SrcDat != nil {
   539  		fileSize = int64(len(pre.SrcDat))
   540  		src.Close()
   541  		src = NewBufReadResetter(pre.SrcDat)
   542  	}
   543  
   544  	var s3params chat1.S3Params
   545  	paramsCh := make(chan struct{})
   546  	g.Go(func() (err error) {
   547  		u.Debug(bgctx, "upload: fetching s3 params")
   548  		u.G().ActivityNotifier.AttachmentUploadStart(bgctx, uid, convID, outboxID)
   549  		if s3params, err = u.ri().GetS3Params(bgctx, chat1.GetS3ParamsArg{
   550  			ConversationID: convID,
   551  			TempCreds:      true,
   552  		}); err != nil {
   553  			return err
   554  		}
   555  		close(paramsCh)
   556  		return nil
   557  	})
   558  	// upload attachment and (optional) preview concurrently
   559  	g.Go(func() (err error) {
   560  		select {
   561  		case <-paramsCh:
   562  		case <-bgctx.Done():
   563  			return bgctx.Err()
   564  		}
   565  
   566  		// set up file to write out encrypted preview to
   567  		var encryptedOut io.Writer
   568  		uf, err := u.uploadFullFile(ctx, pre.BaseMetadata())
   569  		if err != nil {
   570  			u.Debug(bgctx, "upload: failed to create uploaded full file: %s", err)
   571  			encryptedOut = io.Discard
   572  			uf = nil
   573  		} else {
   574  			defer uf.Close()
   575  			encryptedOut = uf
   576  		}
   577  
   578  		u.Debug(bgctx, "upload: uploading assets")
   579  		task := UploadTask{
   580  			S3Params:       s3params,
   581  			Filename:       filename,
   582  			FileSize:       fileSize,
   583  			Plaintext:      src,
   584  			S3Signer:       u.s3signer,
   585  			ConversationID: convID,
   586  			UserID:         uid,
   587  			OutboxID:       outboxID,
   588  			Preview:        false,
   589  			Progress:       progress,
   590  		}
   591  		ures.Object, err = u.store.UploadAsset(bgctx, &task, encryptedOut)
   592  		if err != nil {
   593  			u.Debug(bgctx, "upload: error uploading primary asset to s3: %s", err)
   594  		} else {
   595  			ures.Object.Title = title
   596  			ures.Object.MimeType = pre.ContentType
   597  			ures.Object.Metadata = pre.BaseMetadata()
   598  			if uf != nil {
   599  				if err := u.G().AttachmentURLSrv.GetAttachmentFetcher().PutUploadedAsset(ctx,
   600  					uf.Name(), ures.Object); err != nil {
   601  					u.Debug(bgctx, "upload: failed to put uploaded asset into fetcher: %s", err)
   602  				}
   603  			}
   604  		}
   605  		u.Debug(bgctx, "upload: asset upload complete")
   606  		return err
   607  	})
   608  	if pre.Preview != nil {
   609  		g.Go(func() error {
   610  			select {
   611  			case <-paramsCh:
   612  			case <-bgctx.Done():
   613  				return bgctx.Err()
   614  			}
   615  
   616  			// check to make sure this isn't an emoji conv, and if so just abort
   617  			conv, err := utils.GetUnverifiedConv(bgctx, u.G(), uid, convID, types.InboxSourceDataSourceAll)
   618  			if err != nil {
   619  				return err
   620  			}
   621  			switch conv.GetTopicType() {
   622  			case chat1.TopicType_EMOJI, chat1.TopicType_EMOJICROSS:
   623  				u.Debug(bgctx, "upload: skipping preview upload in emoji conv")
   624  				return nil
   625  			default:
   626  			}
   627  
   628  			// copy the params so as not to mess with the main params above
   629  			previewParams := s3params
   630  
   631  			// set up file to write out encrypted preview to
   632  			var encryptedOut io.Writer
   633  			up, err := u.uploadPreviewFile(ctx)
   634  			if err != nil {
   635  				u.Debug(bgctx, "upload: failed to create uploaded preview file: %s", err)
   636  				encryptedOut = io.Discard
   637  				up = nil
   638  			} else {
   639  				defer up.Close()
   640  				encryptedOut = up
   641  			}
   642  
   643  			// add preview suffix to object key (P in hex)
   644  			// the s3path in gregor is expecting hex here
   645  			previewParams.ObjectKey += "50"
   646  			task := UploadTask{
   647  				S3Params:       previewParams,
   648  				Filename:       filename,
   649  				FileSize:       int64(len(pre.Preview)),
   650  				Plaintext:      NewBufReadResetter(pre.Preview),
   651  				S3Signer:       u.s3signer,
   652  				ConversationID: convID,
   653  				UserID:         uid,
   654  				OutboxID:       outboxID,
   655  				Preview:        true,
   656  			}
   657  			preview, err := u.store.UploadAsset(bgctx, &task, encryptedOut)
   658  			if err == nil {
   659  				ures.Preview = &preview
   660  				ures.Preview.MimeType = pre.PreviewContentType
   661  				ures.Preview.Metadata = pre.PreviewMetadata()
   662  				ures.Preview.Tag = chat1.AssetTag_PRIMARY
   663  				if up != nil {
   664  					if err := u.G().AttachmentURLSrv.GetAttachmentFetcher().PutUploadedAsset(ctx,
   665  						up.Name(), preview); err != nil {
   666  						u.Debug(bgctx, "upload: failed to put uploaded preview asset into fetcher: %s", err)
   667  					}
   668  				}
   669  			} else {
   670  				u.Debug(bgctx, "upload: error uploading preview asset to s3: %s", err)
   671  			}
   672  			u.Debug(bgctx, "upload: preview upload complete")
   673  			return err
   674  		})
   675  	}
   676  
   677  	deferToBackgroundRoutine = true
   678  	go func() {
   679  		defer src.Close()
   680  		var errStr string
   681  		status := types.AttachmentUploaderTaskStatusSuccess
   682  		if err := g.Wait(); err != nil {
   683  			status = types.AttachmentUploaderTaskStatusFailed
   684  			ures.Error = new(string)
   685  			*ures.Error = err.Error()
   686  			errStr = err.Error()
   687  		}
   688  		if err := u.setStatus(bgctx, outboxID, uploaderStatus{
   689  			Status: status,
   690  			Result: ures,
   691  		}); err != nil {
   692  			u.Debug(bgctx, "failed to set status on upload success: %s", err)
   693  		}
   694  		u.Debug(bgctx, "upload: upload complete: status: %v err: %s", status, errStr)
   695  		// Ping Deliverer to notify that some of the message in the outbox might be read to send
   696  		u.G().MessageDeliverer.ForceDeliverLoop(bgctx)
   697  		upload.uploadResult.trigger(ures)
   698  		u.doneUploading(outboxID)
   699  	}()
   700  	return upload.uploadResult, nil
   701  }
   702  
   703  func (u *Uploader) getUploadTempBaseDir(version int) string {
   704  	base := filepath.Join(u.G().GetSharedCacheDir(), uploadedTempsDir)
   705  	// version 0 didn't have the naming scheme ready, so special case it
   706  	if version == 0 {
   707  		return base
   708  	}
   709  	return fmt.Sprintf("%s_v%d", base, version)
   710  }
   711  
   712  func (u *Uploader) getUploadTempDir(version int, outboxID chat1.OutboxID) string {
   713  	return filepath.Join(u.getUploadTempBaseDir(version), outboxID.String())
   714  }
   715  
   716  func (u *Uploader) GetUploadTempFile(ctx context.Context, outboxID chat1.OutboxID, filename string) (string, error) {
   717  	dir := u.getUploadTempDir(u.versionUploaderTemps, outboxID)
   718  	if err := os.MkdirAll(dir, os.ModePerm); err != nil {
   719  		return "", err
   720  	}
   721  	return filepath.Join(dir, filepath.Base(filename)), nil
   722  }
   723  
   724  func (u *Uploader) GetUploadTempSink(ctx context.Context, filename string) (*os.File, chat1.OutboxID, error) {
   725  	obid, err := storage.NewOutboxID()
   726  	if err != nil {
   727  		return nil, nil, err
   728  	}
   729  	filename, err = u.GetUploadTempFile(ctx, obid, filename)
   730  	if err != nil {
   731  		return nil, nil, err
   732  	}
   733  	file, err := os.Create(filename)
   734  	if err != nil {
   735  		return nil, nil, err
   736  	}
   737  	return file, obid, nil
   738  }
   739  
   740  func (u *Uploader) CancelUploadTempFile(ctx context.Context, outboxID chat1.OutboxID) error {
   741  	u.clearTempDirFromOutboxID(ctx, outboxID)
   742  	return nil
   743  }
   744  
   745  func (u *Uploader) getPreviewsDir() string {
   746  	return filepath.Join(u.getBaseDir(), uploadedPreviewsDir)
   747  }
   748  
   749  func (u *Uploader) getFullsDir() string {
   750  	return filepath.Join(u.getBaseDir(), uploadedFullsDir)
   751  }
   752  
   753  func (u *Uploader) OnDbNuke(mctx libkb.MetaContext) error {
   754  	if err := u.previewsLRU.CleanOutOfSync(mctx, u.getPreviewsDir()); err != nil {
   755  		u.Debug(mctx.Ctx(), "unable to run clean for uploadedPreviews: %v", err)
   756  	}
   757  	if err := u.fullsLRU.CleanOutOfSync(mctx, u.getFullsDir()); err != nil {
   758  		u.Debug(mctx.Ctx(), "unable to run clean for uploadedFulls: %v", err)
   759  	}
   760  	return nil
   761  }