github.com/keybase/client/go@v0.0.0-20241007131713-f10651d043c8/kbfs/libkbfs/journal_manager.go (about)

     1  // Copyright 2016 Keybase Inc. All rights reserved.
     2  // Use of this source code is governed by a BSD
     3  // license that can be found in the LICENSE file.
     4  
     5  package libkbfs
     6  
     7  import (
     8  	"fmt"
     9  	"os"
    10  	"path/filepath"
    11  	"regexp"
    12  	"strconv"
    13  	"strings"
    14  	"sync"
    15  	"time"
    16  
    17  	"github.com/keybase/client/go/kbfs/data"
    18  	"github.com/keybase/client/go/kbfs/ioutil"
    19  	"github.com/keybase/client/go/kbfs/kbfsblock"
    20  	"github.com/keybase/client/go/kbfs/kbfscrypto"
    21  	"github.com/keybase/client/go/kbfs/kbfsmd"
    22  	"github.com/keybase/client/go/kbfs/tlf"
    23  	"github.com/keybase/client/go/kbfs/tlfhandle"
    24  	"github.com/keybase/client/go/logger"
    25  	"github.com/keybase/client/go/protocol/keybase1"
    26  	"github.com/pkg/errors"
    27  	"golang.org/x/net/context"
    28  	"golang.org/x/sync/errgroup"
    29  )
    30  
    31  const (
    32  	tlfJournalBrokenFmt = "%s-%d.broken"
    33  )
    34  
    35  var tlfJournalBrokenRegexp = regexp.MustCompile(
    36  	`[/\\]([[:alnum:]]+)-([[:digit:]]+)\.broken$`)
    37  
    38  type journalManagerConfig struct {
    39  	// EnableAuto, if true, means the user has explicitly set its
    40  	// value. If false, then either the user turned it on and then
    41  	// off, or the user hasn't turned it on at all.
    42  	EnableAuto bool
    43  
    44  	// EnableAutoSetByUser means the user has explicitly set the
    45  	// value of EnableAuto (after this field was added).
    46  	EnableAutoSetByUser bool
    47  }
    48  
    49  func (jsc journalManagerConfig) getEnableAuto(currentUID keybase1.UID) (
    50  	enableAuto, enableAutoSetByUser bool) {
    51  	// If EnableAuto is true, the user has explicitly set its value.
    52  	if jsc.EnableAuto {
    53  		return true, true
    54  	}
    55  
    56  	// Otherwise, if EnableAutoSetByUser is true, it means the
    57  	// user has explicitly set the value of EnableAuto (after that
    58  	// field was added).
    59  	if jsc.EnableAutoSetByUser {
    60  		return false, true
    61  	}
    62  
    63  	// Otherwise, if the user hasn't explicitly turned off journaling,
    64  	// it's enabled by default.
    65  	return true, false
    66  }
    67  
    68  // ConflictJournalRecord contains info for TLF journals that are
    69  // currently in conflict on the local device.
    70  type ConflictJournalRecord struct {
    71  	Name           tlf.CanonicalName
    72  	Type           tlf.Type
    73  	Path           string
    74  	ID             tlf.ID
    75  	ServerViewPath keybase1.Path // for cleared conflicts only
    76  	LocalViewPath  keybase1.Path // for cleared conflicts only
    77  }
    78  
    79  // JournalManagerStatus represents the overall status of the
    80  // JournalManager for display in diagnostics. It is suitable for
    81  // encoding directly as JSON.
    82  type JournalManagerStatus struct {
    83  	RootDir             string
    84  	Version             int
    85  	CurrentUID          keybase1.UID
    86  	CurrentVerifyingKey kbfscrypto.VerifyingKey
    87  	EnableAuto          bool
    88  	EnableAutoSetByUser bool
    89  	JournalCount        int
    90  	// The byte counters below are signed because
    91  	// os.FileInfo.Size() is signed. The file counter is signed
    92  	// for consistency.
    93  	StoredBytes       int64
    94  	StoredFiles       int64
    95  	UnflushedBytes    int64
    96  	UnflushedPaths    []string
    97  	EndEstimate       *time.Time
    98  	DiskLimiterStatus interface{}
    99  	Conflicts         []ConflictJournalRecord `json:",omitempty"`
   100  	ClearedConflicts  []ConflictJournalRecord `json:",omitempty"`
   101  }
   102  
   103  // branchChangeListener describes a caller that will get updates via
   104  // the onTLFBranchChange method call when the journal branch changes
   105  // for the given TlfID.  If a new branch has been created, the given
   106  // kbfsmd.BranchID will be something other than kbfsmd.NullBranchID.  If the current
   107  // branch was pruned, it will be kbfsmd.NullBranchID.  If the implementer
   108  // will be accessing the journal, it must do so from another goroutine
   109  // to avoid deadlocks.
   110  type branchChangeListener interface {
   111  	onTLFBranchChange(tlf.ID, kbfsmd.BranchID)
   112  }
   113  
   114  // mdFlushListener describes a caller that will ge updates via the
   115  // onMDFlush metod when an MD is flushed.  If the implementer will be
   116  // accessing the journal, it must do so from another goroutine to
   117  // avoid deadlocks.
   118  type mdFlushListener interface {
   119  	onMDFlush(tlf.ID, kbfsmd.BranchID, kbfsmd.Revision)
   120  }
   121  
   122  type clearedConflictKey struct {
   123  	tlfID tlf.ID
   124  	date  time.Time // the conflict time truncated to be just the date
   125  	num   uint16
   126  }
   127  
   128  type clearedConflictVal struct {
   129  	fakeTlfID tlf.ID
   130  	t         time.Time
   131  }
   132  
   133  // JournalManager is the server that handles write journals. It
   134  // interposes itself in front of BlockServer and MDOps. It uses MDOps
   135  // instead of MDServer because it has to potentially modify the
   136  // RootMetadata passed in, and by the time it hits MDServer it's
   137  // already too late. However, this assumes that all MD ops go through
   138  // MDOps.
   139  //
   140  // The maximum number of characters added to the root dir by a journal
   141  // server journal is 108: 51 for the TLF journal, and 57 for
   142  // everything else.
   143  //
   144  //	/v1/de...-...(53 characters total)...ff(/tlf journal)
   145  type JournalManager struct {
   146  	config     Config
   147  	defaultBWS TLFJournalBackgroundWorkStatus
   148  
   149  	log      traceLogger
   150  	deferLog traceLogger
   151  
   152  	dir string
   153  
   154  	delegateBlockCache      data.BlockCache
   155  	delegateDirtyBlockCache data.DirtyBlockCache
   156  	delegateBlockServer     BlockServer
   157  	delegateMDOps           MDOps
   158  	onBranchChange          branchChangeListener
   159  	onMDFlush               mdFlushListener
   160  
   161  	// Just protects lastQuotaError.
   162  	lastQuotaErrorLock sync.Mutex
   163  	lastQuotaError     time.Time
   164  
   165  	// Just protects lastDiskLimitError.
   166  	lastDiskLimitErrorLock sync.Mutex
   167  	lastDiskLimitError     time.Time
   168  
   169  	// Protects all fields below.
   170  	lock                sync.RWMutex
   171  	currentUID          keybase1.UID
   172  	currentVerifyingKey kbfscrypto.VerifyingKey
   173  	tlfJournals         map[tlf.ID]*tlfJournal
   174  	dirtyOps            map[tlf.ID]uint
   175  	dirtyOpsDone        *sync.Cond
   176  	serverConfig        journalManagerConfig
   177  	// Real TLF ID -> time that conflict was cleared -> fake TLF ID
   178  	clearedConflictTlfs map[clearedConflictKey]clearedConflictVal
   179  	delegateMaker       func(tlf.ID) tlfJournalBWDelegate
   180  }
   181  
   182  func makeJournalManager(
   183  	config Config, log logger.Logger, dir string,
   184  	bcache data.BlockCache, dirtyBcache data.DirtyBlockCache,
   185  	bserver BlockServer, mdOps MDOps, onBranchChange branchChangeListener,
   186  	onMDFlush mdFlushListener,
   187  	bws TLFJournalBackgroundWorkStatus) *JournalManager {
   188  	if len(dir) == 0 {
   189  		panic("journal root path string unexpectedly empty")
   190  	}
   191  	jManager := JournalManager{
   192  		config:                  config,
   193  		defaultBWS:              bws,
   194  		log:                     traceLogger{log},
   195  		deferLog:                traceLogger{log.CloneWithAddedDepth(1)},
   196  		dir:                     dir,
   197  		delegateBlockCache:      bcache,
   198  		delegateDirtyBlockCache: dirtyBcache,
   199  		delegateBlockServer:     bserver,
   200  		delegateMDOps:           mdOps,
   201  		onBranchChange:          onBranchChange,
   202  		onMDFlush:               onMDFlush,
   203  		tlfJournals:             make(map[tlf.ID]*tlfJournal),
   204  		dirtyOps:                make(map[tlf.ID]uint),
   205  		clearedConflictTlfs:     make(map[clearedConflictKey]clearedConflictVal),
   206  	}
   207  	jManager.dirtyOpsDone = sync.NewCond(&jManager.lock)
   208  	return &jManager
   209  }
   210  
   211  func (j *JournalManager) rootPath() string {
   212  	return filepath.Join(j.dir, "v1")
   213  }
   214  
   215  func (j *JournalManager) configPath() string {
   216  	return filepath.Join(j.rootPath(), "config.json")
   217  }
   218  
   219  func (j *JournalManager) readConfig() error {
   220  	return ioutil.DeserializeFromJSONFile(j.configPath(), &j.serverConfig)
   221  }
   222  
   223  func (j *JournalManager) writeConfig() error {
   224  	return ioutil.SerializeToJSONFile(j.serverConfig, j.configPath())
   225  }
   226  
   227  func (j *JournalManager) tlfJournalPathLocked(tlfID tlf.ID) string {
   228  	if j.currentVerifyingKey == (kbfscrypto.VerifyingKey{}) {
   229  		panic("currentVerifyingKey is zero")
   230  	}
   231  
   232  	// We need to generate a unique path for each (UID, device,
   233  	// TLF) tuple. Verifying keys (which are unique to a device)
   234  	// are globally unique, so no need to have the uid in the
   235  	// path. Furthermore, everything after the first two bytes
   236  	// (four characters) is randomly generated, so taking the
   237  	// first 36 characters of the verifying key gives us 16 random
   238  	// bytes (since the first two bytes encode version/type) or
   239  	// 128 random bits, which means that the expected number of
   240  	// devices generated before getting a collision in the first
   241  	// part of the path is 2^64 (see
   242  	// https://en.wikipedia.org/wiki/Birthday_problem#Cast_as_a_collision_problem
   243  	// ).
   244  	//
   245  	// By similar reasoning, for a single device, taking the first
   246  	// 16 characters of the TLF ID gives us 64 random bits, which
   247  	// means that the expected number of TLFs associated to that
   248  	// device before getting a collision in the second part of the
   249  	// path is 2^32.
   250  	shortDeviceIDStr := j.currentVerifyingKey.String()[:36]
   251  	shortTlfIDStr := tlfID.String()[:16]
   252  	dir := fmt.Sprintf("%s-%s", shortDeviceIDStr, shortTlfIDStr)
   253  	return filepath.Join(j.rootPath(), dir)
   254  }
   255  
   256  func (j *JournalManager) getEnableAutoLocked() (
   257  	enableAuto, enableAutoSetByUser bool) {
   258  	return j.serverConfig.getEnableAuto(j.currentUID)
   259  }
   260  
   261  func (j *JournalManager) getConflictIDForHandle(
   262  	tlfID tlf.ID, h *tlfhandle.Handle) (tlf.ID, bool) {
   263  	j.lock.RLock()
   264  	defer j.lock.RUnlock()
   265  	// If the handle represents a local conflict, change the
   266  	// handle's TLF ID to reflect that.
   267  	ci := h.ConflictInfo()
   268  	if ci == nil {
   269  		return tlf.NullID, false
   270  	}
   271  
   272  	if ci.Type != tlf.HandleExtensionLocalConflict {
   273  		return tlf.NullID, false
   274  	}
   275  
   276  	key := clearedConflictKey{
   277  		tlfID: tlfID,
   278  		date:  time.Unix(ci.Date, 0).UTC().Round(0),
   279  		num:   ci.Number,
   280  	}
   281  	val, ok := j.clearedConflictTlfs[key]
   282  	if !ok || val.fakeTlfID == tlf.NullID {
   283  		return tlf.NullID, false
   284  	}
   285  
   286  	return val.fakeTlfID, true
   287  }
   288  
   289  func (j *JournalManager) getTLFJournal(
   290  	tlfID tlf.ID, h *tlfhandle.Handle) (*tlfJournal, bool) {
   291  	getJournalFn := func() (*tlfJournal, bool, bool, bool) {
   292  		j.lock.RLock()
   293  		defer j.lock.RUnlock()
   294  		// Don't create any journals when logged out.
   295  		if j.currentUID.IsNil() {
   296  			return nil, false, false, false
   297  		}
   298  
   299  		tlfJournal, ok := j.tlfJournals[tlfID]
   300  		enableAuto, enableAutoSetByUser := j.getEnableAutoLocked()
   301  		return tlfJournal, enableAuto, enableAutoSetByUser, ok
   302  	}
   303  	tlfJournal, enableAuto, enableAutoSetByUser, ok := getJournalFn()
   304  	if !ok && enableAuto {
   305  		ctx := context.TODO() // plumb through from callers
   306  
   307  		if h == nil {
   308  			// h must always be passed in for MD write operations, so
   309  			// we are always safe in refusing new TLF journals in this
   310  			// case.
   311  			return nil, false
   312  		}
   313  
   314  		// Because of the above handle check, which will happen on
   315  		// every put of a TLF, we will be able to create a journal on
   316  		// the first write that happens after the user becomes a
   317  		// writer for the TLF.
   318  		isWriter, err := IsWriterFromHandle(
   319  			ctx, h, j.config.KBPKI(), j.config, j.currentUID,
   320  			j.currentVerifyingKey)
   321  		if err != nil {
   322  			j.log.CWarningf(ctx, "Couldn't find writership for %s: %+v",
   323  				tlfID, err)
   324  			return nil, false
   325  		}
   326  		if !isWriter {
   327  			return nil, false
   328  		}
   329  
   330  		j.log.CDebugf(ctx, "Enabling a new journal for %s (enableAuto=%t, set by user=%t)",
   331  			tlfID, enableAuto, enableAutoSetByUser)
   332  		err = j.Enable(ctx, tlfID, h, j.defaultBWS)
   333  		if err != nil {
   334  			j.log.CWarningf(ctx, "Couldn't enable journal for %s: %+v", tlfID, err)
   335  			return nil, false
   336  		}
   337  		tlfJournal, _, _, ok = getJournalFn()
   338  	}
   339  	return tlfJournal, ok
   340  }
   341  
   342  func (j *JournalManager) hasTLFJournal(tlfID tlf.ID) bool {
   343  	j.lock.RLock()
   344  	defer j.lock.RUnlock()
   345  	_, ok := j.tlfJournals[tlfID]
   346  	return ok
   347  }
   348  
   349  func (j *JournalManager) getHandleForJournal(
   350  	ctx context.Context, tj *tlfJournal, tlfID tlf.ID) (
   351  	*tlfhandle.Handle, error) {
   352  	bid, err := tj.getBranchID()
   353  	if err != nil {
   354  		return nil, err
   355  	}
   356  
   357  	head, err := tj.getMDHead(ctx, bid)
   358  	if err != nil {
   359  		return nil, err
   360  	}
   361  
   362  	if head == (ImmutableBareRootMetadata{}) {
   363  		return nil, nil
   364  	}
   365  
   366  	headBareHandle, err := head.MakeBareTlfHandleWithExtra()
   367  	if err != nil {
   368  		return nil, err
   369  	}
   370  
   371  	return tlfhandle.MakeHandleWithTlfID(
   372  		ctx, headBareHandle, tlfID.Type(), j.config.KBPKI(),
   373  		j.config.KBPKI(), tlfID, j.config.OfflineAvailabilityForID(tlfID))
   374  }
   375  
   376  func (j *JournalManager) makeFBOForJournal(
   377  	ctx context.Context, tj *tlfJournal, tlfID tlf.ID,
   378  	branch data.BranchName) error {
   379  	handle, err := j.getHandleForJournal(ctx, tj, tlfID)
   380  	if err != nil {
   381  		return err
   382  	}
   383  	if handle == nil {
   384  		return nil
   385  	}
   386  
   387  	_, _, err = j.config.KBFSOps().GetRootNode(ctx, handle, branch)
   388  	return err
   389  }
   390  
   391  // MakeFBOsForExistingJournals creates folderBranchOps objects for all
   392  // existing, non-empty journals.  This is useful to initialize the
   393  // unflushed edit history, for example.  It returns a wait group that
   394  // the caller can use to determine when all the FBOs have been
   395  // initialized.  If the caller is not going to wait on the group, it
   396  // should provide a context that won't be canceled before the wait
   397  // group is finished.
   398  func (j *JournalManager) MakeFBOsForExistingJournals(
   399  	ctx context.Context) *sync.WaitGroup {
   400  	var wg sync.WaitGroup
   401  
   402  	j.lock.Lock()
   403  	defer j.lock.Unlock()
   404  	for tlfID, tj := range j.tlfJournals {
   405  		wg.Add(1)
   406  		tlfID := tlfID
   407  		tj := tj
   408  		go func() {
   409  			ctx := CtxWithRandomIDReplayable(
   410  				context.Background(), CtxFBOIDKey, CtxFBOOpID, j.log)
   411  
   412  			// Turn off tracker popups.
   413  			ctx, err := tlfhandle.MakeExtendedIdentify(
   414  				ctx, keybase1.TLFIdentifyBehavior_KBFS_INIT)
   415  			if err != nil {
   416  				j.log.CWarningf(ctx, "Error making extended identify: %+v", err)
   417  			}
   418  
   419  			defer wg.Done()
   420  			j.log.CDebugf(ctx,
   421  				"Initializing FBO for non-empty journal: %s", tlfID)
   422  
   423  			branch := data.MasterBranch
   424  			if tj.overrideTlfID != tlf.NullID {
   425  				// Find the conflict key.
   426  				for k, v := range j.clearedConflictTlfs {
   427  					if tj.overrideTlfID != v.fakeTlfID {
   428  						continue
   429  					}
   430  
   431  					ext, err := tlf.NewHandleExtension(
   432  						tlf.HandleExtensionLocalConflict, k.num, "", k.date)
   433  					if err != nil {
   434  						j.log.CWarningf(ctx, "Error making extension: %+v", err)
   435  						continue
   436  					}
   437  
   438  					branch = data.MakeConflictBranchNameFromExtension(ext)
   439  				}
   440  			}
   441  
   442  			err = j.makeFBOForJournal(ctx, tj, tlfID, branch)
   443  			if err != nil {
   444  				j.log.CWarningf(ctx,
   445  					"Error when making FBO for existing journal for %s: "+
   446  						"%+v", tlfID, err)
   447  			}
   448  
   449  			// The popups and errors were suppressed, but any errors would
   450  			// have been logged.  So just close out the extended identify.  If
   451  			// the user accesses the TLF directly, another proper identify
   452  			// should happen that shows errors.
   453  			_ = tlfhandle.GetExtendedIdentify(ctx).GetTlfBreakAndClose()
   454  		}()
   455  	}
   456  	return &wg
   457  }
   458  
   459  func (j *JournalManager) makeJournalForConflictTlfLocked(
   460  	ctx context.Context, dir string, tlfID tlf.ID,
   461  	chargedTo keybase1.UserOrTeamID) (*tlfJournal, tlf.ID, time.Time, error) {
   462  	// If this is a bak directory representing a
   463  	// moved-away conflicts branch, we should assign it a
   464  	// fake TLF ID.
   465  	matches := tlfJournalBakRegexp.FindStringSubmatch(dir)
   466  	if len(matches) == 0 {
   467  		return nil, tlf.ID{}, time.Time{},
   468  			errors.Errorf("%s is not a backup conflict dir", dir)
   469  	}
   470  
   471  	unixNano, err := strconv.ParseInt(matches[2], 10, 64)
   472  	if err != nil {
   473  		return nil, tlf.ID{}, time.Time{}, err
   474  	}
   475  
   476  	fakeTlfID, err := tlf.MakeRandomID(tlfID.Type())
   477  	if err != nil {
   478  		return nil, tlf.ID{}, time.Time{}, err
   479  	}
   480  
   481  	var delegate tlfJournalBWDelegate
   482  	if j.delegateMaker != nil {
   483  		delegate = j.delegateMaker(fakeTlfID)
   484  	}
   485  
   486  	tj, err := makeTLFJournal(
   487  		ctx, j.currentUID, j.currentVerifyingKey, dir,
   488  		tlfID, chargedTo, tlfJournalConfigAdapter{j.config},
   489  		j.delegateBlockServer, TLFJournalBackgroundWorkPaused, delegate,
   490  		j.onBranchChange, j.onMDFlush, j.config.DiskLimiter(), fakeTlfID)
   491  	if err != nil {
   492  		return nil, tlf.ID{}, time.Time{}, err
   493  	}
   494  
   495  	return tj, fakeTlfID, time.Unix(0, unixNano), nil
   496  }
   497  
   498  func (j *JournalManager) insertConflictJournalLocked(
   499  	ctx context.Context, tj *tlfJournal, fakeTlfID tlf.ID, t time.Time) {
   500  	const dateFormat = "2006-01-02"
   501  	dateStr := t.UTC().Format(dateFormat)
   502  	date, err := time.Parse(dateFormat, dateStr)
   503  	if err != nil {
   504  		panic(err.Error())
   505  	}
   506  	date = date.UTC().Round(0)
   507  
   508  	key := clearedConflictKey{
   509  		tlfID: tj.tlfID,
   510  		date:  date,
   511  	}
   512  
   513  	val := clearedConflictVal{
   514  		fakeTlfID: fakeTlfID,
   515  		t:         t,
   516  	}
   517  
   518  	// Figure out what number conflict this should be.
   519  	num := uint16(1)
   520  	toDel := make([]clearedConflictKey, 0)
   521  	toAdd := make(map[clearedConflictKey]clearedConflictVal)
   522  	for otherKey, otherVal := range j.clearedConflictTlfs {
   523  		if otherKey.tlfID != tj.tlfID || !otherKey.date.Equal(date) {
   524  			continue
   525  		}
   526  
   527  		// Increase the number for each conflict that happened before
   528  		// this one; increase any existing numbers that happened after
   529  		// it.
   530  		if otherVal.t.Before(t) {
   531  			num++
   532  		} else {
   533  			toDel = append(toDel, otherKey)
   534  			otherKey.num++
   535  			toAdd[otherKey] = otherVal
   536  		}
   537  	}
   538  	key.num = num
   539  
   540  	for _, k := range toDel {
   541  		delete(j.clearedConflictTlfs, k)
   542  	}
   543  	for k, v := range toAdd {
   544  		j.clearedConflictTlfs[k] = v
   545  	}
   546  
   547  	j.clearedConflictTlfs[key] = val
   548  	j.tlfJournals[fakeTlfID] = tj
   549  	j.log.CDebugf(ctx, "Made conflict journal for %s, real "+
   550  		"TLF ID = %s, fake TLF ID = %s, date = %s, num = %d",
   551  		tj.dir, tj.tlfID, fakeTlfID, date, num)
   552  }
   553  
   554  // EnableExistingJournals turns on the write journal for all TLFs for
   555  // the given (UID, device) tuple (with the device identified by its
   556  // verifying key) with an existing journal. Any returned error means
   557  // that the JournalManager remains in the same state as it was before.
   558  //
   559  // Once this is called, this must not be called again until
   560  // shutdownExistingJournals is called.
   561  func (j *JournalManager) EnableExistingJournals(
   562  	ctx context.Context, currentUID keybase1.UID,
   563  	currentVerifyingKey kbfscrypto.VerifyingKey,
   564  	bws TLFJournalBackgroundWorkStatus) (err error) {
   565  	j.log.CDebugf(ctx, "Enabling existing journals (%s)", bws)
   566  	defer func() {
   567  		if err != nil {
   568  			j.deferLog.CDebugf(ctx,
   569  				"Error when enabling existing journals: %+v",
   570  				err)
   571  		}
   572  	}()
   573  
   574  	if currentUID == keybase1.UID("") {
   575  		return errors.New("Current UID is empty")
   576  	}
   577  	if currentVerifyingKey == (kbfscrypto.VerifyingKey{}) {
   578  		return errors.New("Current verifying key is empty")
   579  	}
   580  
   581  	// TODO: We should also look up journals from other
   582  	// users/devices so that we can take into account their
   583  	// journal usage.
   584  
   585  	j.lock.Lock()
   586  	defer j.lock.Unlock()
   587  
   588  	if j.currentUID == currentUID {
   589  		// The user is not changing, so nothing needs to be done.
   590  		return nil
   591  	} else if j.currentUID != keybase1.UID("") {
   592  		return errors.Errorf("Trying to set current UID from %s to %s",
   593  			j.currentUID, currentUID)
   594  	}
   595  	if j.currentVerifyingKey != (kbfscrypto.VerifyingKey{}) {
   596  		return errors.Errorf(
   597  			"Trying to set current verifying key from %s to %s",
   598  			j.currentVerifyingKey, currentVerifyingKey)
   599  	}
   600  
   601  	err = j.readConfig()
   602  	switch {
   603  	case ioutil.IsNotExist(err):
   604  		// Config file doesn't exist, so write out a default one.
   605  		err := j.writeConfig()
   606  		if err != nil {
   607  			return err
   608  		}
   609  	case err != nil:
   610  		return err
   611  	}
   612  
   613  	// Need to set it here since tlfJournalPathLocked and
   614  	// enableLocked depend on it.
   615  	j.currentUID = currentUID
   616  	j.currentVerifyingKey = currentVerifyingKey
   617  
   618  	enableSucceeded := false
   619  	defer func() {
   620  		// Revert to a clean state if the enable doesn't
   621  		// succeed, either due to a panic or error.
   622  		if !enableSucceeded {
   623  			j.shutdownExistingJournalsLocked(ctx)
   624  		}
   625  	}()
   626  
   627  	fileInfos, err := ioutil.ReadDir(j.rootPath())
   628  	if ioutil.IsNotExist(err) {
   629  		enableSucceeded = true
   630  		return nil
   631  	} else if err != nil {
   632  		return err
   633  	}
   634  
   635  	eg, groupCtx := errgroup.WithContext(ctx)
   636  
   637  	fileCh := make(chan os.FileInfo, len(fileInfos))
   638  	type journalRet struct {
   639  		id      tlf.ID
   640  		journal *tlfJournal
   641  	}
   642  	journalCh := make(chan journalRet, len(fileInfos))
   643  	var conflictLock sync.Mutex
   644  	worker := func() error {
   645  		for fi := range fileCh {
   646  			name := fi.Name()
   647  			if !fi.IsDir() {
   648  				j.log.CDebugf(groupCtx, "Skipping file %q", name)
   649  				continue
   650  			}
   651  
   652  			dir := filepath.Join(j.rootPath(), name)
   653  
   654  			// Skip directories that have already been marked as broken.
   655  			matches := tlfJournalBrokenRegexp.FindStringSubmatch(dir)
   656  			if len(matches) > 0 {
   657  				j.log.CDebugf(groupCtx, "Skipping broken dir %q", name)
   658  				continue
   659  			}
   660  
   661  			// Skip directories that don't have an info file at all.
   662  			_, err := os.Lstat(getTLFJournalInfoFilePath(dir))
   663  			switch {
   664  			case err == nil:
   665  			case os.IsNotExist(err):
   666  				j.log.CDebugf(
   667  					groupCtx, "Skipping non-TLF dir %q", name)
   668  				continue
   669  			default:
   670  				j.log.CDebugf(
   671  					groupCtx, "Error stat'ing info file in dir %q: %+v",
   672  					name, err)
   673  				continue
   674  			}
   675  
   676  			uid, key, tlfID, chargedTo, err := readTLFJournalInfoFile(dir)
   677  			if err != nil {
   678  				idParts := strings.Split(name, "-")
   679  				newDirName := fmt.Sprintf(
   680  					tlfJournalBrokenFmt, idParts[len(idParts)-1],
   681  					j.config.Clock().Now().UnixNano())
   682  				fullDirName := filepath.Join(j.rootPath(), newDirName)
   683  
   684  				j.log.CDebugf(
   685  					groupCtx, "Renaming broken dir %q to %q due to error: %+v",
   686  					name, newDirName, err)
   687  
   688  				err := os.Rename(dir, fullDirName)
   689  				if err != nil {
   690  					j.log.CDebugf(
   691  						groupCtx, "Error renaming broken dir %q: %+v",
   692  						name, err)
   693  				}
   694  				continue
   695  			}
   696  
   697  			if uid != currentUID {
   698  				j.log.CDebugf(
   699  					groupCtx, "Skipping dir %q due to mismatched UID %s",
   700  					name, uid)
   701  				continue
   702  			}
   703  
   704  			if key != currentVerifyingKey {
   705  				j.log.CDebugf(
   706  					groupCtx, "Skipping dir %q due to mismatched key %s",
   707  					name, uid)
   708  				continue
   709  			}
   710  
   711  			expectedDir := j.tlfJournalPathLocked(tlfID)
   712  			if dir != expectedDir {
   713  				tj, fakeTlfID, t, err := j.makeJournalForConflictTlfLocked(
   714  					groupCtx, dir, tlfID, chargedTo)
   715  				if err != nil {
   716  					j.log.CDebugf(
   717  						groupCtx, "Skipping misnamed dir %s: %+v", dir, err)
   718  					continue
   719  				}
   720  
   721  				// Take a lock while inserting the conflict journal
   722  				// (even though we already have `journalLock`), since
   723  				// multiple workers could be running at once and we
   724  				// need to protect the cleared conflct TLF map from
   725  				// concurrent access.
   726  				conflictLock.Lock()
   727  				j.insertConflictJournalLocked(groupCtx, tj, fakeTlfID, t)
   728  				conflictLock.Unlock()
   729  				continue
   730  			}
   731  
   732  			// Allow enable even if dirty, since any dirty writes
   733  			// in flight are most likely for another user.
   734  			tj, err := j.enableLocked(groupCtx, tlfID, chargedTo, bws, true)
   735  			if err != nil {
   736  				// Don't treat per-TLF errors as fatal.
   737  				j.log.CWarningf(
   738  					groupCtx,
   739  					"Error when enabling existing journal for %s: %+v",
   740  					tlfID, err)
   741  				continue
   742  			}
   743  
   744  			// Delete any empty journals so they don't clutter up the
   745  			// directory, until the TLF is accessed again.
   746  			blockEntryCount, mdEntryCount, err := tj.getJournalEntryCounts()
   747  			if err != nil {
   748  				tj.shutdown(groupCtx)
   749  				// Don't treat per-TLF errors as fatal.
   750  				j.log.CWarningf(
   751  					groupCtx,
   752  					"Error when getting status of existing journal for %s: %+v",
   753  					tlfID, err)
   754  				continue
   755  			}
   756  			if blockEntryCount == 0 && mdEntryCount == 0 {
   757  				j.log.CDebugf(groupCtx, "Nuking empty journal for %s", tlfID)
   758  				tj.shutdown(groupCtx)
   759  				os.RemoveAll(dir)
   760  				continue
   761  			}
   762  
   763  			journalCh <- journalRet{tlfID, tj}
   764  		}
   765  		return nil
   766  	}
   767  
   768  	// Initialize many TLF journals at once to overlap disk latency as
   769  	// much as possible.
   770  	numWorkers := 100
   771  	if numWorkers > len(fileInfos) {
   772  		numWorkers = len(fileInfos)
   773  	}
   774  	for i := 0; i < numWorkers; i++ {
   775  		eg.Go(worker)
   776  	}
   777  
   778  	for _, fi := range fileInfos {
   779  		fileCh <- fi
   780  	}
   781  	close(fileCh)
   782  
   783  	err = eg.Wait()
   784  	if err != nil {
   785  		// None of the workers return an error so this should never
   786  		// happen...
   787  		return err
   788  	}
   789  	close(journalCh)
   790  
   791  	for r := range journalCh {
   792  		j.tlfJournals[r.id] = r.journal
   793  	}
   794  
   795  	j.log.CDebugf(ctx, "Done enabling journals")
   796  
   797  	enableSucceeded = true
   798  	return nil
   799  }
   800  
   801  // enabledLocked returns an enabled journal; it is the caller's
   802  // responsibility to add it to `j.tlfJournals`.  This allows this
   803  // method to be called in parallel during initialization, if desired.
   804  func (j *JournalManager) enableLocked(
   805  	ctx context.Context, tlfID tlf.ID, chargedTo keybase1.UserOrTeamID,
   806  	bws TLFJournalBackgroundWorkStatus, allowEnableIfDirty bool) (
   807  	tj *tlfJournal, err error) {
   808  	j.log.CDebugf(ctx, "Enabling journal for %s (%s)", tlfID, bws)
   809  	defer func() {
   810  		if err != nil {
   811  			j.deferLog.CDebugf(ctx,
   812  				"Error when enabling journal for %s: %+v",
   813  				tlfID, err)
   814  		}
   815  	}()
   816  
   817  	if j.currentUID == keybase1.UID("") {
   818  		return nil, errors.New("Current UID is empty")
   819  	}
   820  	if j.currentVerifyingKey == (kbfscrypto.VerifyingKey{}) {
   821  		return nil, errors.New("Current verifying key is empty")
   822  	}
   823  
   824  	if tj, ok := j.tlfJournals[tlfID]; ok {
   825  		err = tj.enable()
   826  		if err != nil {
   827  			return nil, err
   828  		}
   829  		return tj, nil
   830  	}
   831  
   832  	err = func() error {
   833  		if j.dirtyOps[tlfID] > 0 {
   834  			return errors.Errorf("Can't enable journal for %s while there "+
   835  				"are outstanding dirty ops", tlfID)
   836  		}
   837  		if j.delegateDirtyBlockCache.IsAnyDirty(tlfID) {
   838  			return errors.Errorf("Can't enable journal for %s while there "+
   839  				"are any dirty blocks outstanding", tlfID)
   840  		}
   841  		return nil
   842  	}()
   843  	if err != nil {
   844  		if !allowEnableIfDirty {
   845  			return nil, err
   846  		}
   847  
   848  		j.log.CWarningf(ctx,
   849  			"Got ignorable error on journal enable, and proceeding anyway: %+v",
   850  			err)
   851  	}
   852  
   853  	var delegate tlfJournalBWDelegate
   854  	if j.delegateMaker != nil {
   855  		delegate = j.delegateMaker(tlfID)
   856  	}
   857  
   858  	tlfDir := j.tlfJournalPathLocked(tlfID)
   859  	tj, err = makeTLFJournal(
   860  		ctx, j.currentUID, j.currentVerifyingKey, tlfDir,
   861  		tlfID, chargedTo, tlfJournalConfigAdapter{j.config},
   862  		j.delegateBlockServer, bws, delegate, j.onBranchChange, j.onMDFlush,
   863  		j.config.DiskLimiter(),
   864  		tlf.NullID)
   865  	if err != nil {
   866  		return nil, err
   867  	}
   868  
   869  	return tj, nil
   870  }
   871  
   872  // Enable turns on the write journal for the given TLF.  If h is nil,
   873  // it will be attempted to be fetched from the remote MD server.
   874  func (j *JournalManager) Enable(ctx context.Context, tlfID tlf.ID,
   875  	h *tlfhandle.Handle, bws TLFJournalBackgroundWorkStatus) (err error) {
   876  	j.lock.Lock()
   877  	defer j.lock.Unlock()
   878  	chargedTo := j.currentUID.AsUserOrTeam()
   879  	if tlfID.Type() == tlf.SingleTeam {
   880  		if h == nil {
   881  			// Any path that creates a single-team TLF journal should
   882  			// also provide a handle.  If not, we'd have to fetch it
   883  			// from the server, which isn't a trusted path.
   884  			return errors.Errorf(
   885  				"No handle provided for single-team TLF %s", tlfID)
   886  		}
   887  
   888  		chargedTo = h.FirstResolvedWriter()
   889  		if tid := chargedTo.AsTeamOrBust(); tid.IsSubTeam() {
   890  			// We can't charge to subteams; find the root team.
   891  			rootID, err := j.config.KBPKI().GetTeamRootID(
   892  				ctx, tid, j.config.OfflineAvailabilityForID(tlfID))
   893  			if err != nil {
   894  				return err
   895  			}
   896  			chargedTo = rootID.AsUserOrTeam()
   897  		}
   898  	}
   899  	tj, err := j.enableLocked(ctx, tlfID, chargedTo, bws, false)
   900  	if err != nil {
   901  		return err
   902  	}
   903  	j.tlfJournals[tlfID] = tj
   904  	return nil
   905  }
   906  
   907  // EnableAuto turns on the write journal for all TLFs, even new ones,
   908  // persistently.
   909  func (j *JournalManager) EnableAuto(ctx context.Context) error {
   910  	j.lock.Lock()
   911  	defer j.lock.Unlock()
   912  	if j.serverConfig.EnableAuto {
   913  		// Nothing to do.
   914  		return nil
   915  	}
   916  
   917  	j.log.CDebugf(ctx, "Enabling auto-journaling")
   918  	j.serverConfig.EnableAuto = true
   919  	j.serverConfig.EnableAutoSetByUser = true
   920  	return j.writeConfig()
   921  }
   922  
   923  // DisableAuto turns off automatic write journal for any
   924  // newly-accessed TLFs.  Existing journaled TLFs need to be disabled
   925  // manually.
   926  func (j *JournalManager) DisableAuto(ctx context.Context) error {
   927  	j.lock.Lock()
   928  	defer j.lock.Unlock()
   929  	if enabled, _ := j.getEnableAutoLocked(); !enabled {
   930  		// Nothing to do.
   931  		return nil
   932  	}
   933  
   934  	j.log.CDebugf(ctx, "Disabling auto-journaling")
   935  	j.serverConfig.EnableAuto = false
   936  	j.serverConfig.EnableAutoSetByUser = true
   937  	return j.writeConfig()
   938  }
   939  
   940  func (j *JournalManager) dirtyOpStart(tlfID tlf.ID) {
   941  	j.lock.Lock()
   942  	defer j.lock.Unlock()
   943  	j.dirtyOps[tlfID]++
   944  }
   945  
   946  func (j *JournalManager) dirtyOpEnd(tlfID tlf.ID) {
   947  	j.lock.Lock()
   948  	defer j.lock.Unlock()
   949  	if j.dirtyOps[tlfID] == 0 {
   950  		panic("Trying to end a dirty op when count is 0")
   951  	}
   952  	j.dirtyOps[tlfID]--
   953  	if j.dirtyOps[tlfID] == 0 {
   954  		delete(j.dirtyOps, tlfID)
   955  	}
   956  	if len(j.dirtyOps) == 0 {
   957  		j.dirtyOpsDone.Broadcast()
   958  	}
   959  }
   960  
   961  // PauseBackgroundWork pauses the background work goroutine, if it's
   962  // not already paused.
   963  func (j *JournalManager) PauseBackgroundWork(ctx context.Context, tlfID tlf.ID) {
   964  	j.log.CDebugf(ctx, "Signaling pause for %s", tlfID)
   965  	if tlfJournal, ok := j.getTLFJournal(tlfID, nil); ok {
   966  		tlfJournal.pauseBackgroundWork()
   967  		return
   968  	}
   969  
   970  	j.log.CDebugf(ctx,
   971  		"Could not find journal for %s; dropping pause signal",
   972  		tlfID)
   973  }
   974  
   975  // ResumeBackgroundWork resumes the background work goroutine, if it's
   976  // not already resumed.
   977  func (j *JournalManager) ResumeBackgroundWork(ctx context.Context, tlfID tlf.ID) {
   978  	j.log.CDebugf(ctx, "Signaling resume for %s", tlfID)
   979  	if tlfJournal, ok := j.getTLFJournal(tlfID, nil); ok {
   980  		tlfJournal.resumeBackgroundWork()
   981  		return
   982  	}
   983  
   984  	j.log.CDebugf(ctx,
   985  		"Could not find journal for %s; dropping resume signal",
   986  		tlfID)
   987  }
   988  
   989  // Flush flushes the write journal for the given TLF.
   990  func (j *JournalManager) Flush(ctx context.Context, tlfID tlf.ID) (err error) {
   991  	j.log.CDebugf(ctx, "Flushing journal for %s", tlfID)
   992  	if tlfJournal, ok := j.getTLFJournal(tlfID, nil); ok {
   993  		// TODO: do we want to plumb lc through here as well?
   994  		return tlfJournal.flush(ctx)
   995  	}
   996  
   997  	j.log.CDebugf(ctx, "Journal not enabled for %s", tlfID)
   998  	return nil
   999  }
  1000  
  1001  // Wait blocks until the write journal has finished flushing
  1002  // everything.  It is essentially the same as Flush() when the journal
  1003  // is enabled and unpaused, except that it is safe to cancel the
  1004  // context without leaving the journal in a partially-flushed state.
  1005  // It does not wait for any conflicts or squashes resulting from
  1006  // flushing the data currently in the journal.
  1007  func (j *JournalManager) Wait(ctx context.Context, tlfID tlf.ID) (err error) {
  1008  	j.log.CDebugf(ctx, "Waiting on journal for %s", tlfID)
  1009  	if tlfJournal, ok := j.getTLFJournal(tlfID, nil); ok {
  1010  		return tlfJournal.wait(ctx)
  1011  	}
  1012  
  1013  	j.log.CDebugf(ctx, "Journal not enabled for %s", tlfID)
  1014  	return nil
  1015  }
  1016  
  1017  // WaitForCompleteFlush blocks until the write journal has finished
  1018  // flushing everything.  Unlike `Wait()`, it also waits for any
  1019  // conflicts or squashes detected during each flush attempt.
  1020  func (j *JournalManager) WaitForCompleteFlush(
  1021  	ctx context.Context, tlfID tlf.ID) (err error) {
  1022  	j.log.CDebugf(ctx, "Finishing single op for %s", tlfID)
  1023  	if tlfJournal, ok := j.getTLFJournal(tlfID, nil); ok {
  1024  		return tlfJournal.waitForCompleteFlush(ctx)
  1025  	}
  1026  
  1027  	j.log.CDebugf(ctx, "Journal not enabled for %s", tlfID)
  1028  	return nil
  1029  }
  1030  
  1031  // FinishSingleOp lets the write journal know that the application has
  1032  // finished a single op, and then blocks until the write journal has
  1033  // finished flushing everything.  If this folder is not being flushed
  1034  // in single op mode, this call is equivalent to
  1035  // `WaitForCompleteFlush`.
  1036  func (j *JournalManager) FinishSingleOp(ctx context.Context, tlfID tlf.ID,
  1037  	lc *keybase1.LockContext, priority keybase1.MDPriority) (err error) {
  1038  	j.log.CDebugf(ctx, "Finishing single op for %s", tlfID)
  1039  	if tlfJournal, ok := j.getTLFJournal(tlfID, nil); ok {
  1040  		return tlfJournal.finishSingleOp(ctx, lc, priority)
  1041  	}
  1042  
  1043  	j.log.CDebugf(ctx, "Journal not enabled for %s", tlfID)
  1044  	return nil
  1045  }
  1046  
  1047  // Disable turns off the write journal for the given TLF.
  1048  func (j *JournalManager) Disable(ctx context.Context, tlfID tlf.ID) (
  1049  	wasEnabled bool, err error) {
  1050  	j.log.CDebugf(ctx, "Disabling journal for %s", tlfID)
  1051  	defer func() {
  1052  		if err != nil {
  1053  			j.deferLog.CDebugf(ctx,
  1054  				"Error when disabling journal for %s: %+v",
  1055  				tlfID, err)
  1056  		}
  1057  	}()
  1058  
  1059  	j.lock.Lock()
  1060  	defer j.lock.Unlock()
  1061  	tlfJournal, ok := j.tlfJournals[tlfID]
  1062  	if !ok {
  1063  		j.log.CDebugf(ctx, "Journal doesn't exist for %s", tlfID)
  1064  		return false, nil
  1065  	}
  1066  
  1067  	if j.dirtyOps[tlfID] > 0 {
  1068  		return false, errors.Errorf("Can't disable journal for %s while there "+
  1069  			"are outstanding dirty ops", tlfID)
  1070  	}
  1071  	if j.delegateDirtyBlockCache.IsAnyDirty(tlfID) {
  1072  		return false, errors.Errorf("Can't disable journal for %s while there "+
  1073  			"are any dirty blocks outstanding", tlfID)
  1074  	}
  1075  
  1076  	// Disable the journal.  Note that we don't bother deleting the
  1077  	// journal from j.tlfJournals, to avoid cases where something
  1078  	// keeps it around doing background work or re-enables it, at the
  1079  	// same time JournalManager creates a new journal for the same TLF.
  1080  	wasEnabled, err = tlfJournal.disable()
  1081  	if err != nil {
  1082  		return false, err
  1083  	}
  1084  
  1085  	if wasEnabled {
  1086  		j.log.CDebugf(ctx, "Disabled journal for %s", tlfID)
  1087  	}
  1088  	return wasEnabled, nil
  1089  }
  1090  
  1091  func (j *JournalManager) blockCache() journalBlockCache {
  1092  	return journalBlockCache{j, j.delegateBlockCache}
  1093  }
  1094  
  1095  func (j *JournalManager) dirtyBlockCache(
  1096  	journalCache data.DirtyBlockCache) journalDirtyBlockCache {
  1097  	return journalDirtyBlockCache{j, j.delegateDirtyBlockCache, journalCache}
  1098  }
  1099  
  1100  func (j *JournalManager) blockServer() journalBlockServer {
  1101  	return journalBlockServer{j, j.delegateBlockServer, false}
  1102  }
  1103  
  1104  func (j *JournalManager) mdOps() journalMDOps {
  1105  	return journalMDOps{j.delegateMDOps, j}
  1106  }
  1107  
  1108  func (j *JournalManager) maybeReturnOverQuotaError(
  1109  	usedQuotaBytes, quotaBytes int64) error {
  1110  	if usedQuotaBytes <= quotaBytes {
  1111  		return nil
  1112  	}
  1113  
  1114  	j.lastQuotaErrorLock.Lock()
  1115  	defer j.lastQuotaErrorLock.Unlock()
  1116  
  1117  	now := j.config.Clock().Now()
  1118  	// Return OverQuota errors only occasionally, so we don't spam
  1119  	// the keybase daemon with notifications. (See
  1120  	// PutBlockCheckQuota in block_util.go.)
  1121  	const overQuotaDuration = time.Minute
  1122  	if now.Sub(j.lastQuotaError) < overQuotaDuration {
  1123  		return nil
  1124  	}
  1125  
  1126  	j.lastQuotaError = now
  1127  	return kbfsblock.ServerErrorOverQuota{
  1128  		Usage:     usedQuotaBytes,
  1129  		Limit:     quotaBytes,
  1130  		Throttled: false,
  1131  	}
  1132  }
  1133  
  1134  func (j *JournalManager) maybeMakeDiskLimitErrorReportable(
  1135  	err *ErrDiskLimitTimeout) error {
  1136  	j.lastDiskLimitErrorLock.Lock()
  1137  	defer j.lastDiskLimitErrorLock.Unlock()
  1138  
  1139  	now := j.config.Clock().Now()
  1140  	// Return DiskLimit errors only occasionally, so we don't spam
  1141  	// the keybase daemon with notifications. (See
  1142  	// PutBlockCheckLimitErrs in block_util.go.)
  1143  	const overDiskLimitDuration = time.Minute
  1144  	if now.Sub(j.lastDiskLimitError) < overDiskLimitDuration {
  1145  		return err
  1146  	}
  1147  
  1148  	err.reportable = true
  1149  	j.lastDiskLimitError = now
  1150  	return err
  1151  }
  1152  
  1153  func (j *JournalManager) getJournalsInConflictLocked(ctx context.Context) (
  1154  	current, cleared []ConflictJournalRecord, err error) {
  1155  	for _, tlfJournal := range j.tlfJournals {
  1156  		if tlfJournal.overrideTlfID != tlf.NullID {
  1157  			continue
  1158  		}
  1159  		isConflict, err := tlfJournal.isOnConflictBranch()
  1160  		if err != nil {
  1161  			return nil, nil, err
  1162  		}
  1163  		if !isConflict {
  1164  			continue
  1165  		}
  1166  
  1167  		handle, err := j.getHandleForJournal(ctx, tlfJournal, tlfJournal.tlfID)
  1168  		if err != nil {
  1169  			return nil, nil, err
  1170  		}
  1171  		if handle == nil {
  1172  			continue
  1173  		}
  1174  
  1175  		current = append(current, ConflictJournalRecord{
  1176  			Name: handle.GetCanonicalName(),
  1177  			Type: handle.Type(),
  1178  			Path: handle.GetCanonicalPath(),
  1179  			ID:   tlfJournal.tlfID,
  1180  		})
  1181  	}
  1182  
  1183  	for key, val := range j.clearedConflictTlfs {
  1184  		fakeTlfID := val.fakeTlfID
  1185  		if fakeTlfID == tlf.NullID {
  1186  			continue
  1187  		}
  1188  		tlfJournal := j.tlfJournals[fakeTlfID]
  1189  
  1190  		handle, err := j.getHandleForJournal(ctx, tlfJournal, tlfJournal.tlfID)
  1191  		if err != nil {
  1192  			return nil, nil, err
  1193  		}
  1194  		if handle == nil {
  1195  			continue
  1196  		}
  1197  		serverViewPath := handle.GetProtocolPath()
  1198  
  1199  		ext, err := tlf.NewHandleExtension(
  1200  			tlf.HandleExtensionLocalConflict, key.num, "", key.date)
  1201  		if err != nil {
  1202  			return nil, nil, err
  1203  		}
  1204  		handle, err = handle.WithUpdatedConflictInfo(j.config.Codec(), ext)
  1205  		if err != nil {
  1206  			return nil, nil, err
  1207  		}
  1208  
  1209  		cleared = append(cleared, ConflictJournalRecord{
  1210  			Name:           handle.GetCanonicalName(),
  1211  			Type:           handle.Type(),
  1212  			Path:           handle.GetCanonicalPath(),
  1213  			ID:             tlfJournal.tlfID,
  1214  			ServerViewPath: serverViewPath,
  1215  			LocalViewPath:  handle.GetProtocolPath(),
  1216  		})
  1217  	}
  1218  
  1219  	return current, cleared, nil
  1220  }
  1221  
  1222  // GetJournalsInConflict returns records for each TLF journal that
  1223  // currently has a conflict.
  1224  func (j *JournalManager) GetJournalsInConflict(ctx context.Context) (
  1225  	current, cleared []ConflictJournalRecord, err error) {
  1226  	j.lock.RLock()
  1227  	defer j.lock.RUnlock()
  1228  	return j.getJournalsInConflictLocked(ctx)
  1229  }
  1230  
  1231  // GetFoldersSummary returns the TLFs with journals in conflict, and
  1232  // the number of TLFs that have unuploaded data.
  1233  func (j *JournalManager) GetFoldersSummary() (
  1234  	tlfsInConflict []tlf.ID, numUploadingTlfs int, err error) {
  1235  	j.lock.RLock()
  1236  	defer j.lock.RUnlock()
  1237  
  1238  	for _, tlfJournal := range j.tlfJournals {
  1239  		if tlfJournal.overrideTlfID != tlf.NullID {
  1240  			continue
  1241  		}
  1242  		isConflict, err := tlfJournal.isOnConflictBranch()
  1243  		if err != nil {
  1244  			return nil, 0, err
  1245  		}
  1246  		if isConflict {
  1247  			tlfsInConflict = append(tlfsInConflict, tlfJournal.tlfID)
  1248  		}
  1249  
  1250  		_, _, unflushedBytes, err := tlfJournal.getByteCounts()
  1251  		if err != nil {
  1252  			return nil, 0, err
  1253  		}
  1254  
  1255  		if unflushedBytes > 0 {
  1256  			numUploadingTlfs++
  1257  		}
  1258  	}
  1259  
  1260  	return tlfsInConflict, numUploadingTlfs, nil
  1261  }
  1262  
  1263  // Status returns a JournalManagerStatus object suitable for
  1264  // diagnostics.  It also returns a list of TLF IDs which have journals
  1265  // enabled.
  1266  func (j *JournalManager) Status(
  1267  	ctx context.Context) (JournalManagerStatus, []tlf.ID) {
  1268  	j.lock.RLock()
  1269  	defer j.lock.RUnlock()
  1270  	var totalStoredBytes, totalStoredFiles, totalUnflushedBytes int64
  1271  	tlfIDs := make([]tlf.ID, 0, len(j.tlfJournals))
  1272  	for _, tlfJournal := range j.tlfJournals {
  1273  		storedBytes, storedFiles, unflushedBytes, err :=
  1274  			tlfJournal.getByteCounts()
  1275  		if err != nil {
  1276  			j.log.CWarningf(ctx,
  1277  				"Couldn't calculate stored bytes/stored files/unflushed bytes for %s: %+v",
  1278  				tlfJournal.tlfID, err)
  1279  		}
  1280  		totalStoredBytes += storedBytes
  1281  		totalStoredFiles += storedFiles
  1282  		totalUnflushedBytes += unflushedBytes
  1283  		tlfIDs = append(tlfIDs, tlfJournal.tlfID)
  1284  	}
  1285  	enableAuto, enableAutoSetByUser := j.getEnableAutoLocked()
  1286  	currentConflicts, clearedConflicts, err :=
  1287  		j.getJournalsInConflictLocked(ctx)
  1288  	if err != nil {
  1289  		j.log.CWarningf(ctx, "Couldn't get conflict journals: %+v", err)
  1290  		currentConflicts = nil
  1291  		clearedConflicts = nil
  1292  	}
  1293  	return JournalManagerStatus{
  1294  		RootDir:             j.rootPath(),
  1295  		Version:             1,
  1296  		CurrentUID:          j.currentUID,
  1297  		CurrentVerifyingKey: j.currentVerifyingKey,
  1298  		EnableAuto:          enableAuto,
  1299  		EnableAutoSetByUser: enableAutoSetByUser,
  1300  		JournalCount:        len(tlfIDs),
  1301  		StoredBytes:         totalStoredBytes,
  1302  		StoredFiles:         totalStoredFiles,
  1303  		UnflushedBytes:      totalUnflushedBytes,
  1304  		DiskLimiterStatus: j.config.DiskLimiter().getStatus(
  1305  			ctx, j.currentUID.AsUserOrTeam()),
  1306  		Conflicts:        currentConflicts,
  1307  		ClearedConflicts: clearedConflicts,
  1308  	}, tlfIDs
  1309  }
  1310  
  1311  // JournalStatus returns a TLFServerStatus object for the given TLF
  1312  // suitable for diagnostics.
  1313  func (j *JournalManager) JournalStatus(tlfID tlf.ID) (
  1314  	TLFJournalStatus, error) {
  1315  	tlfJournal, ok := j.getTLFJournal(tlfID, nil)
  1316  	if !ok {
  1317  		return TLFJournalStatus{},
  1318  			errors.Errorf("Journal not enabled for %s", tlfID)
  1319  	}
  1320  
  1321  	return tlfJournal.getJournalStatus()
  1322  }
  1323  
  1324  // JournalEnabled returns true if the given TLF ID has a journal
  1325  // enabled for it.
  1326  func (j *JournalManager) JournalEnabled(tlfID tlf.ID) bool {
  1327  	_, ok := j.getTLFJournal(tlfID, nil)
  1328  	return ok
  1329  }
  1330  
  1331  // JournalStatusWithPaths returns a TLFServerStatus object for the
  1332  // given TLF suitable for diagnostics, including paths for all the
  1333  // unflushed entries.
  1334  func (j *JournalManager) JournalStatusWithPaths(ctx context.Context,
  1335  	tlfID tlf.ID, cpp chainsPathPopulator) (TLFJournalStatus, error) {
  1336  	tlfJournal, ok := j.getTLFJournal(tlfID, nil)
  1337  	if !ok {
  1338  		return TLFJournalStatus{},
  1339  			errors.Errorf("Journal not enabled for %s", tlfID)
  1340  	}
  1341  
  1342  	return tlfJournal.getJournalStatusWithPaths(ctx, cpp)
  1343  }
  1344  
  1345  // MoveAway moves the current conflict branch to a new journal
  1346  // directory for the given TLF ID, and exposes it under a different
  1347  // favorite name in the folder list.
  1348  func (j *JournalManager) MoveAway(ctx context.Context, tlfID tlf.ID) error {
  1349  	tlfJournal, ok := j.getTLFJournal(tlfID, nil)
  1350  	if !ok {
  1351  		return errJournalNotAvailable
  1352  	}
  1353  
  1354  	err := tlfJournal.wait(ctx)
  1355  	if err != nil {
  1356  		return err
  1357  	}
  1358  	newDir, err := tlfJournal.moveAway(ctx)
  1359  	if err != nil {
  1360  		return err
  1361  	}
  1362  
  1363  	j.lock.Lock()
  1364  	defer j.lock.Unlock()
  1365  	tj, fakeTlfID, t, err := j.makeJournalForConflictTlfLocked(
  1366  		ctx, newDir, tlfID, tlfJournal.chargedTo)
  1367  	if err != nil {
  1368  		return err
  1369  	}
  1370  	j.insertConflictJournalLocked(ctx, tj, fakeTlfID, t)
  1371  	j.config.SubscriptionManagerPublisher().PublishChange(
  1372  		keybase1.SubscriptionTopic_FAVORITES)
  1373  	j.config.SubscriptionManagerPublisher().PublishChange(
  1374  		keybase1.SubscriptionTopic_FILES_TAB_BADGE)
  1375  	return j.config.KeybaseService().NotifyFavoritesChanged(ctx)
  1376  }
  1377  
  1378  func (j *JournalManager) deleteJournal(
  1379  	ctx context.Context, tlfID tlf.ID, clearConflict bool) (err error) {
  1380  	var journalDir string
  1381  	defer func() {
  1382  		if err != nil {
  1383  			return
  1384  		}
  1385  		// Remove the journal dir outside of the lock, since it could
  1386  		// take some time if the conflict branch was large.
  1387  		err = ioutil.RemoveAll(journalDir)
  1388  	}()
  1389  
  1390  	j.lock.Lock()
  1391  	defer j.lock.Unlock()
  1392  
  1393  	tlfJournal, ok := j.tlfJournals[tlfID]
  1394  	if !ok {
  1395  		return errJournalNotAvailable
  1396  	}
  1397  
  1398  	if clearConflict {
  1399  		found := false
  1400  		for k, v := range j.clearedConflictTlfs {
  1401  			if tlfID != v.fakeTlfID {
  1402  				continue
  1403  			}
  1404  			// Nullify the TLF ID in the cleared conflict map, so we can
  1405  			// preserve the number of the deleted conflict TLF (so that
  1406  			// future conflicts on this same date get a new number), but
  1407  			// without having it show up in the favorites list.
  1408  			v.fakeTlfID = tlf.NullID
  1409  			j.clearedConflictTlfs[k] = v
  1410  			found = true
  1411  			break
  1412  		}
  1413  
  1414  		if !found {
  1415  			return errors.Errorf("%s is not a cleared conflict journal", tlfID)
  1416  		}
  1417  	}
  1418  
  1419  	// Shut down the journal and remove from the cleared conflicts map.
  1420  	tlfJournal.shutdown(ctx)
  1421  	delete(j.tlfJournals, tlfID)
  1422  	journalDir = tlfJournal.dir
  1423  	return nil
  1424  }
  1425  
  1426  // FinishResolvingConflict shuts down the TLF journal for a cleared
  1427  // conflict, and removes its storage from the local disk.
  1428  func (j *JournalManager) FinishResolvingConflict(
  1429  	ctx context.Context, fakeTlfID tlf.ID) (err error) {
  1430  	return j.deleteJournal(ctx, fakeTlfID, true)
  1431  }
  1432  
  1433  // DeleteJournal shuts down a TLF journal, and removes its storage
  1434  // from the local disk.
  1435  func (j *JournalManager) DeleteJournal(
  1436  	ctx context.Context, tlfID tlf.ID) (err error) {
  1437  	return j.deleteJournal(ctx, tlfID, false)
  1438  }
  1439  
  1440  // shutdownExistingJournalsLocked shuts down all write journals, sets
  1441  // the current UID and verifying key to zero, and returns once all
  1442  // shutdowns are complete. It is safe to call multiple times in a row,
  1443  // and once this is called, EnableExistingJournals may be called
  1444  // again.
  1445  func (j *JournalManager) shutdownExistingJournalsLocked(ctx context.Context) {
  1446  	for len(j.dirtyOps) > 0 {
  1447  		j.log.CDebugf(ctx,
  1448  			"Waiting for %d TLFS with dirty ops before shutting down "+
  1449  				"existing journals...", len(j.dirtyOps))
  1450  		j.dirtyOpsDone.Wait()
  1451  	}
  1452  
  1453  	j.log.CDebugf(ctx, "Shutting down existing journals")
  1454  
  1455  	for _, tlfJournal := range j.tlfJournals {
  1456  		tlfJournal.shutdown(ctx)
  1457  	}
  1458  
  1459  	j.tlfJournals = make(map[tlf.ID]*tlfJournal)
  1460  	j.currentUID = keybase1.UID("")
  1461  	j.currentVerifyingKey = kbfscrypto.VerifyingKey{}
  1462  	j.clearedConflictTlfs = make(map[clearedConflictKey]clearedConflictVal)
  1463  }
  1464  
  1465  // shutdownExistingJournals shuts down all write journals, sets the
  1466  // current UID and verifying key to zero, and returns once all
  1467  // shutdowns are complete. It is safe to call multiple times in a row,
  1468  // and once this is called, EnableExistingJournals may be called
  1469  // again.
  1470  func (j *JournalManager) shutdownExistingJournals(ctx context.Context) {
  1471  	j.lock.Lock()
  1472  	defer j.lock.Unlock()
  1473  	j.shutdownExistingJournalsLocked(ctx)
  1474  }
  1475  
  1476  func (j *JournalManager) shutdown(ctx context.Context) {
  1477  	j.log.CDebugf(ctx, "Shutting down journal")
  1478  	j.lock.Lock()
  1479  	defer j.lock.Unlock()
  1480  	for _, tlfJournal := range j.tlfJournals {
  1481  		tlfJournal.shutdown(ctx)
  1482  	}
  1483  
  1484  	// Leave all the tlfJournals in j.tlfJournals, so that any
  1485  	// access to them errors out instead of mutating the journal.
  1486  }
  1487  
  1488  func (j *JournalManager) setDelegateMaker(f func(tlf.ID) tlfJournalBWDelegate) {
  1489  	j.lock.Lock()
  1490  	defer j.lock.Unlock()
  1491  	j.delegateMaker = f
  1492  }