github.com/keybase/client/go@v0.0.0-20241007131713-f10651d043c8/kbfs/libkbfs/init.go (about)

     1  // Copyright 2016 Keybase Inc. All rights reserved.
     2  // Use of this source code is governed by a BSD
     3  // license that can be found in the LICENSE file.
     4  
     5  package libkbfs
     6  
     7  import (
     8  	"context"
     9  	"errors"
    10  	"flag"
    11  	"fmt"
    12  	"os"
    13  	"os/signal"
    14  	"path/filepath"
    15  	"runtime/pprof"
    16  	"strings"
    17  	"syscall"
    18  	"time"
    19  
    20  	"github.com/keybase/client/go/kbconst"
    21  	"github.com/keybase/client/go/kbfs/data"
    22  	"github.com/keybase/client/go/kbfs/kbfscrypto"
    23  	"github.com/keybase/client/go/kbfs/kbfsmd"
    24  	"github.com/keybase/client/go/kbfs/libkey"
    25  	"github.com/keybase/client/go/libkb"
    26  	"github.com/keybase/client/go/logger"
    27  	"github.com/keybase/client/go/protocol/keybase1"
    28  	"github.com/keybase/go-framed-msgpack-rpc/rpc"
    29  )
    30  
    31  const (
    32  	// InitDefaultString is the normal mode for when KBFS data will be
    33  	// read and written.
    34  	InitDefaultString string = "default"
    35  	// InitMinimalString is for when KBFS will only be used as a MD
    36  	// lookup layer (e.g., for chat on mobile).
    37  	InitMinimalString = "minimal"
    38  	// InitSingleOpString is for when KBFS will only be used for a
    39  	// single logical operation (e.g., as a git remote helper).
    40  	InitSingleOpString = "singleOp"
    41  	// InitConstrainedString is for when KBFS will use constrained
    42  	// resources.
    43  	InitConstrainedString = "constrained"
    44  	// InitMemoryLimitedString is for when KBFS will use memory limited
    45  	// resources.
    46  	InitMemoryLimitedString = "memoryLimited"
    47  	// InitTestSearchString is for when KBFS will index synced TLFs.
    48  	InitTestSearchString = "testSearch"
    49  	// InitSingleOpWithQRString is for when KBFS will only be used for
    50  	// a single logical operation (e.g., as a git remote helper), with
    51  	// QR enabled.
    52  	InitSingleOpWithQRString = "singleOpQR"
    53  )
    54  
    55  // CtxInitTagKey is the type used for unique context tags for KBFS init.
    56  type CtxInitTagKey int
    57  
    58  const (
    59  	// CtxInitKey is the type of the tag for unique operation IDs
    60  	// for KBFS init.
    61  	CtxInitKey CtxInitTagKey = iota
    62  )
    63  
    64  // CtxInitID is the display name for the unique operation
    65  // init ID tag.
    66  const CtxInitID = "KBFSINIT"
    67  
    68  // AdditionalProtocolCreator creates an additional protocol.
    69  type AdditionalProtocolCreator func(Context, Config) (rpc.Protocol, error)
    70  
    71  const (
    72  	configModeStr                  = "kbfs.mode"
    73  	configBlockCacheMemMaxBytesStr = "kbfs.block_cache.mem_max_bytes"
    74  	configBlockCacheDiskMaxFracStr = "kbfs.block_cache.disk_max_fraction"
    75  	configBlockCacheSyncMaxFracStr = "kbfs.block_cache.sync_max_fraction"
    76  )
    77  
    78  // InitParams contains the initialization parameters for Init(). It is
    79  // usually filled in by the flags parser passed into AddFlags().
    80  type InitParams struct {
    81  	// Whether to print debug messages.
    82  	Debug bool
    83  
    84  	// If non-empty, the host:port of the block server. If empty,
    85  	// a default value is used depending on the run mode. Can also
    86  	// be "memory" for an in-memory test server or
    87  	// "dir:/path/to/dir" for an on-disk test server.
    88  	BServerAddr string
    89  
    90  	// If non-empty the host:port of the metadata server. If
    91  	// empty, a default value is used depending on the run mode.
    92  	// Can also be "memory" for an in-memory test server or
    93  	// "dir:/path/to/dir" for an on-disk test server.
    94  	MDServerAddr string
    95  
    96  	// If non-zero, specifies the capacity (in bytes) of the block cache. If
    97  	// zero, the capacity is set using getDefaultBlockCacheCapacity().
    98  	CleanBlockCacheCapacity uint64
    99  
   100  	// Fake local user name.
   101  	LocalUser string
   102  
   103  	// Where to put favorites. Has an effect only when LocalUser
   104  	// is non-empty, in which case it must be either "memory" or
   105  	// "dir:/path/to/dir".
   106  	LocalFavoriteStorage string
   107  
   108  	// TLFValidDuration is the duration that TLFs are valid
   109  	// before marked for lazy revalidation.
   110  	TLFValidDuration time.Duration
   111  
   112  	// MetadataVersion is the default version of metadata to use
   113  	// when creating new metadata.
   114  	MetadataVersion kbfsmd.MetadataVer
   115  
   116  	// BlockCryptVersion is the encryption version to use when
   117  	// encrypting new blocks.
   118  	BlockCryptVersion kbfscrypto.EncryptionVer
   119  
   120  	// LogToFile if true, logs to a default file location.
   121  	LogToFile bool
   122  
   123  	// LogFileConfig tells us where to log and rotation config.
   124  	LogFileConfig logger.LogFileConfig
   125  
   126  	// TLFJournalBackgroundWorkStatus is the status to use to
   127  	// pass into JournalManager.enableJournaling. Only has an effect when
   128  	// EnableJournal is non-empty.
   129  	TLFJournalBackgroundWorkStatus TLFJournalBackgroundWorkStatus
   130  
   131  	// AdditionalProtocolCreators are for adding additional protocols that we
   132  	// should handle for service to call in.
   133  	AdditionalProtocolCreators []AdditionalProtocolCreator
   134  
   135  	// EnableJournal enables journaling.
   136  	EnableJournal bool
   137  
   138  	// DiskCacheMode specifies which mode to start the disk cache.
   139  	DiskCacheMode DiskCacheMode
   140  
   141  	// StorageRoot, if non-empty, points to a local directory to put its local
   142  	// databases for things like the journal or disk cache.
   143  	StorageRoot string
   144  
   145  	// BGFlushPeriod indicates how long to wait for a batch to fill up
   146  	// before syncing a set of changes on a TLF to the servers.
   147  	BGFlushPeriod time.Duration
   148  
   149  	// BGFlushDirOpBatchSize indicates how many directory operations
   150  	// in a TLF should be batched together in a single background
   151  	// flush.
   152  	BGFlushDirOpBatchSize int
   153  
   154  	// Mode describes how KBFS should initialize itself.
   155  	Mode string
   156  
   157  	// DiskBlockCacheFraction indicates what fraction of free space on the disk
   158  	// is allowed to be occupied by the KBFS disk block cache.
   159  	DiskBlockCacheFraction float64
   160  
   161  	// SyncBlockCacheFraction indicates what fraction of free space on the disk
   162  	// is allowed to be occupied by the KBFS sync block cache for offline use.
   163  	SyncBlockCacheFraction float64
   164  }
   165  
   166  // defaultBServer returns the default value for the -bserver flag.
   167  func defaultBServer(ctx Context) string {
   168  	switch ctx.GetRunMode() {
   169  	case kbconst.DevelRunMode:
   170  		return memoryAddr
   171  	case kbconst.StagingRunMode:
   172  		return `
   173  			bserver-0.dev.keybase.io:443,bserver-1.dev.keybase.io:443`
   174  	case kbconst.ProductionRunMode:
   175  		return `
   176  			bserver-0.kbfs.keybaseapi.com:443,bserver-1.kbfs.keybaseapi.com:443;
   177  			bserver-0.kbfs.keybase.io:443,bserver-1.kbfs.keybase.io:443`
   178  	default:
   179  		return ""
   180  	}
   181  }
   182  
   183  // defaultMDServer returns the default value for the -mdserver flag.
   184  func defaultMDServer(ctx Context) string {
   185  	switch ctx.GetRunMode() {
   186  	case kbconst.DevelRunMode:
   187  		return memoryAddr
   188  	case kbconst.StagingRunMode:
   189  		return `
   190  			mdserver-0.dev.keybase.io:443,mdserver-1.dev.keybase.io:443`
   191  	case kbconst.ProductionRunMode:
   192  		return `
   193  			mdserver-0.kbfs.keybaseapi.com:443,mdserver-1.kbfs.keybaseapi.com:443;
   194  			mdserver-0.kbfs.keybase.io:443,mdserver-1.kbfs.keybase.io:443`
   195  	default:
   196  		return ""
   197  	}
   198  }
   199  
   200  // defaultMetadataVersion returns the default metadata version per run mode.
   201  func defaultMetadataVersion(ctx Context) kbfsmd.MetadataVer {
   202  	switch ctx.GetRunMode() {
   203  	case kbconst.DevelRunMode:
   204  		return kbfsmd.ImplicitTeamsVer
   205  	case kbconst.StagingRunMode:
   206  		return kbfsmd.ImplicitTeamsVer
   207  	case kbconst.ProductionRunMode:
   208  		return kbfsmd.ImplicitTeamsVer
   209  	default:
   210  		return kbfsmd.ImplicitTeamsVer
   211  	}
   212  }
   213  
   214  func defaultLogPath(ctx Context) string {
   215  	return filepath.Join(ctx.GetLogDir(), libkb.KBFSLogFileName)
   216  }
   217  
   218  // DefaultInitParams returns default init params
   219  func DefaultInitParams(ctx Context) InitParams {
   220  	journalEnv := os.Getenv("KBFS_DEFAULT_ENABLE_JOURNAL_VALUE")
   221  	if journalEnv == "" {
   222  		journalEnv = "true"
   223  	}
   224  	return InitParams{
   225  		Debug:             BoolForString(os.Getenv("KBFS_DEBUG")),
   226  		BServerAddr:       defaultBServer(ctx),
   227  		MDServerAddr:      defaultMDServer(ctx),
   228  		TLFValidDuration:  tlfValidDurationDefault,
   229  		MetadataVersion:   defaultMetadataVersion(ctx),
   230  		BlockCryptVersion: kbfscrypto.EncryptionSecretboxWithKeyNonce,
   231  		LogFileConfig: logger.LogFileConfig{
   232  			MaxAge:       30 * 24 * time.Hour,
   233  			MaxSize:      128 * 1024 * 1024,
   234  			MaxKeepFiles: 3,
   235  		},
   236  		TLFJournalBackgroundWorkStatus: TLFJournalBackgroundWorkEnabled,
   237  		StorageRoot:                    ctx.GetDataDir(),
   238  		BGFlushPeriod:                  bgFlushPeriodDefault,
   239  		BGFlushDirOpBatchSize:          bgFlushDirOpBatchSizeDefault,
   240  		EnableJournal:                  BoolForString(journalEnv),
   241  		DiskCacheMode:                  DiskCacheModeLocal,
   242  		Mode:                           "",
   243  	}
   244  }
   245  
   246  // AddFlagsWithDefaults adds libkbfs flags to the given FlagSet, given
   247  // a set of default flags. Returns an InitParams that will be filled
   248  // in once the given FlagSet is parsed.
   249  func AddFlagsWithDefaults(
   250  	flags *flag.FlagSet, defaultParams InitParams,
   251  	defaultLogPath string) *InitParams {
   252  	var params InitParams
   253  	flags.BoolVar(&params.Debug, "debug", defaultParams.Debug,
   254  		"Print debug messages")
   255  
   256  	flags.StringVar(&params.BServerAddr, "bserver", defaultParams.BServerAddr,
   257  		"host:port of the block server, 'memory', or 'dir:/path/to/dir'")
   258  	flags.StringVar(&params.MDServerAddr, "mdserver",
   259  		defaultParams.MDServerAddr,
   260  		"host:port of the metadata server, 'memory', or 'dir:/path/to/dir'")
   261  	flags.StringVar(&params.LocalUser, "localuser", defaultParams.LocalUser,
   262  		"fake local user")
   263  	flags.StringVar(&params.LocalFavoriteStorage, "local-fav-storage",
   264  		defaultParams.LocalFavoriteStorage,
   265  		"where to put favorites; used only when -localuser is set, then must "+
   266  			"either be 'memory' or 'dir:/path/to/dir'")
   267  	flags.DurationVar(&params.TLFValidDuration, "tlf-valid",
   268  		defaultParams.TLFValidDuration,
   269  		"time tlfs are valid before redoing identification")
   270  	flags.BoolVar(&params.LogToFile, "log-to-file", defaultParams.LogToFile,
   271  		fmt.Sprintf("Log to default file: %s", defaultLogPath))
   272  	flags.StringVar(&params.LogFileConfig.Path, "log-file", "",
   273  		"Path to log file")
   274  	flags.DurationVar(&params.LogFileConfig.MaxAge, "log-file-max-age",
   275  		defaultParams.LogFileConfig.MaxAge,
   276  		"Maximum age of a log file before rotation")
   277  	params.LogFileConfig.MaxSize = defaultParams.LogFileConfig.MaxSize
   278  	flags.Var(SizeFlag{&params.LogFileConfig.MaxSize}, "log-file-max-size",
   279  		"Maximum size of a log file before rotation")
   280  	// The default is to *DELETE* old log files for kbfs.
   281  	flags.IntVar(&params.LogFileConfig.MaxKeepFiles, "log-file-max-keep-files",
   282  		defaultParams.LogFileConfig.MaxKeepFiles, "Maximum number of log "+
   283  			"files for this service, older ones are deleted. 0 for infinite.")
   284  	flags.Uint64Var(&params.CleanBlockCacheCapacity, "clean-bcache-cap",
   285  		defaultParams.CleanBlockCacheCapacity,
   286  		"If non-zero, specify the capacity of clean block cache. If zero, "+
   287  			"the capacity is set based on system RAM.")
   288  	flags.StringVar(&params.StorageRoot, "storage-root",
   289  		defaultParams.StorageRoot, "Specifies where Keybase will store its "+
   290  			"local databases for the journal and disk cache.")
   291  	params.DiskCacheMode = defaultParams.DiskCacheMode
   292  	flags.Var(&params.DiskCacheMode, "disk-cache-mode",
   293  		"Sets the mode for the disk cache. If 'local', then it uses a "+
   294  			"subdirectory of -storage-root to store the cache. If 'remote', "+
   295  			"then it connects to the local KBFS instance and delegates disk "+
   296  			"cache operations to it.")
   297  	flags.BoolVar(&params.EnableJournal, "enable-journal",
   298  		defaultParams.EnableJournal, "Enables write journaling for TLFs.")
   299  
   300  	// No real need to enable setting
   301  	// params.TLFJournalBackgroundWorkStatus via a flag.
   302  	params.TLFJournalBackgroundWorkStatus =
   303  		defaultParams.TLFJournalBackgroundWorkStatus
   304  
   305  	flags.DurationVar(&params.BGFlushPeriod, "sync-batch-period",
   306  		defaultParams.BGFlushPeriod,
   307  		"The amount of time to wait before syncing data in a TLF, if the "+
   308  			"batch size doesn't fill up.")
   309  	flags.IntVar(&params.BGFlushDirOpBatchSize, "sync-batch-size",
   310  		defaultParams.BGFlushDirOpBatchSize,
   311  		"The number of unflushed directory operations in a TLF that will "+
   312  			"trigger an immediate data sync.")
   313  
   314  	flags.IntVar((*int)(&params.MetadataVersion), "md-version",
   315  		int(defaultParams.MetadataVersion),
   316  		"Metadata version to use when creating new metadata")
   317  	flags.IntVar((*int)(&params.BlockCryptVersion), "block-crypt-version",
   318  		int(defaultParams.BlockCryptVersion),
   319  		"Encryption version to use when encrypting new blocks")
   320  	flags.StringVar(&params.Mode, "mode", defaultParams.Mode,
   321  		fmt.Sprintf("Overall initialization mode for KBFS, indicating how "+
   322  			"heavy-weight it can be (%s, %s, %s, %s or %s)", InitDefaultString,
   323  			InitMinimalString, InitSingleOpString, InitConstrainedString,
   324  			InitMemoryLimitedString))
   325  
   326  	flags.Float64Var(&params.DiskBlockCacheFraction,
   327  		"disk-block-cache-fraction", defaultParams.DiskBlockCacheFraction,
   328  		"The portion of the free disk space that KBFS will use for caching ")
   329  
   330  	flags.Float64Var(&params.SyncBlockCacheFraction,
   331  		"sync-block-cache-fraction", defaultParams.SyncBlockCacheFraction,
   332  		"The portion of the free disk space that KBFS will use for offline storage")
   333  
   334  	return &params
   335  }
   336  
   337  // AddFlags adds libkbfs flags to the given FlagSet. Returns an
   338  // InitParams that will be filled in once the given FlagSet is parsed.
   339  func AddFlags(flags *flag.FlagSet, ctx Context) *InitParams {
   340  	return AddFlagsWithDefaults(
   341  		flags, DefaultInitParams(ctx), defaultLogPath(ctx))
   342  }
   343  
   344  // GetRemoteUsageString returns a string describing the flags to use
   345  // to run against remote KBFS servers.
   346  func GetRemoteUsageString() string {
   347  	return `    [-debug]
   348      [-bserver=host:port] [-mdserver=host:port]
   349      [-log-to-file] [-log-file=path/to/file] [-clean-bcache-cap=0]`
   350  }
   351  
   352  // GetLocalUsageString returns a string describing the flags to use to
   353  // run in a local testing environment.
   354  func GetLocalUsageString() string {
   355  	return `    [-debug]
   356      [-bserver=(memory | dir:/path/to/dir | host:port)]
   357      [-mdserver=(memory | dir:/path/to/dir | host:port)]
   358      [-localuser=<user>]
   359      [-local-fav-storage=(memory | dir:/path/to/dir)]
   360      [-log-to-file] [-log-file=path/to/file] [-clean-bcache-cap=0]`
   361  }
   362  
   363  // GetDefaultsUsageString returns a string describing the default
   364  // values of flags based on the run mode.
   365  func GetDefaultsUsageString(ctx Context) string {
   366  	runMode := ctx.GetRunMode()
   367  	defaultBServer := defaultBServer(ctx)
   368  	defaultMDServer := defaultMDServer(ctx)
   369  	return fmt.Sprintf(`  (KEYBASE_RUN_MODE=%s)
   370    -bserver=%s
   371    -mdserver=%s`,
   372  		runMode, defaultBServer, defaultMDServer)
   373  }
   374  
   375  const memoryAddr = "memory"
   376  
   377  const dirAddrPrefix = "dir:"
   378  
   379  func parseRootDir(addr string) (string, bool) {
   380  	if !strings.HasPrefix(addr, dirAddrPrefix) {
   381  		return "", false
   382  	}
   383  	serverRootDir := addr[len(dirAddrPrefix):]
   384  	if len(serverRootDir) == 0 {
   385  		return "", false
   386  	}
   387  	return serverRootDir, true
   388  }
   389  
   390  func makeMDServer(kbCtx Context, config Config, mdserverAddr string,
   391  	rpcLogFactory rpc.LogFactory, log logger.Logger) (
   392  	MDServer, error) {
   393  	if mdserverAddr == memoryAddr {
   394  		log.Debug("Using in-memory mdserver")
   395  		// local in-memory MD server
   396  		return NewMDServerMemory(mdServerLocalConfigAdapter{config})
   397  	}
   398  
   399  	if len(mdserverAddr) == 0 {
   400  		return nil, errors.New("Empty MD server address")
   401  	}
   402  
   403  	if serverRootDir, ok := parseRootDir(mdserverAddr); ok {
   404  		log.Debug("Using on-disk mdserver at %s", serverRootDir)
   405  		// local persistent MD server
   406  		return MakeDiskMDServer(config, serverRootDir)
   407  	}
   408  
   409  	remote, err := rpc.ParsePrioritizedRoundRobinRemote(mdserverAddr)
   410  	if err != nil {
   411  		return nil, err
   412  	}
   413  	// remote MD server. this can't fail. reconnection attempts
   414  	// will be automatic.
   415  	log.Debug("Using remote mdserver %s", remote)
   416  	mdServer := NewMDServerRemote(kbCtx, config, remote, rpcLogFactory)
   417  	return mdServer, nil
   418  }
   419  
   420  func makeKeyServer(
   421  	config Config, keyserverAddr string, log logger.Logger) (
   422  	libkey.KeyServer, error) {
   423  	keyOpsConfig := keyOpsConfigWrapper{config}
   424  	if keyserverAddr == memoryAddr {
   425  		log.Debug("Using in-memory keyserver")
   426  		// local in-memory key server
   427  		return libkey.NewKeyServerMemory(keyOpsConfig, log)
   428  	}
   429  
   430  	if len(keyserverAddr) == 0 {
   431  		return nil, errors.New("Empty key server address")
   432  	}
   433  
   434  	if serverRootDir, ok := parseRootDir(keyserverAddr); ok {
   435  		log.Debug("Using on-disk keyserver at %s", serverRootDir)
   436  		// local persistent key server
   437  		keyPath := filepath.Join(serverRootDir, "kbfs_key")
   438  		return libkey.NewKeyServerDir(keyOpsConfig, log, keyPath)
   439  	}
   440  
   441  	log.Debug("Using remote keyserver %s (same as mdserver)", keyserverAddr)
   442  	// currently the MD server also acts as the key server.
   443  	keyServer, ok := config.MDServer().(libkey.KeyServer)
   444  	if !ok {
   445  		return nil, errors.New("MD server is not a key server")
   446  	}
   447  	return keyServer, nil
   448  }
   449  
   450  func makeBlockServer(kbCtx Context, config Config, bserverAddr string,
   451  	rpcLogFactory rpc.LogFactory,
   452  	log logger.Logger) (BlockServer, error) {
   453  	if bserverAddr == memoryAddr {
   454  		log.Debug("Using in-memory bserver")
   455  		bserverLog := config.MakeLogger("BSM")
   456  		// local in-memory block server
   457  		return NewBlockServerMemory(bserverLog), nil
   458  	}
   459  
   460  	if len(bserverAddr) == 0 {
   461  		return nil, errors.New("Empty block server address")
   462  	}
   463  
   464  	if serverRootDir, ok := parseRootDir(bserverAddr); ok {
   465  		log.Debug("Using on-disk bserver at %s", serverRootDir)
   466  		// local persistent block server
   467  		return MakeDiskBlockServer(config, serverRootDir), nil
   468  	}
   469  
   470  	remote, err := rpc.ParsePrioritizedRoundRobinRemote(bserverAddr)
   471  	if err != nil {
   472  		return nil, err
   473  	}
   474  	log.Debug("Using remote bserver %s", remote)
   475  	return NewBlockServerRemote(kbCtx, config, remote, rpcLogFactory), nil
   476  }
   477  
   478  // InitLogWithPrefix sets up logging switching to a log file if
   479  // necessary, given a prefix and a default log path.  Returns a valid
   480  // logger even on error, which are non-fatal, thus errors from this
   481  // function may be ignored.  Possible errors are logged to the logger
   482  // returned.
   483  func InitLogWithPrefix(
   484  	params InitParams, ctx Context, prefix string,
   485  	defaultLogPath string) (logger.Logger, error) {
   486  	var err error
   487  
   488  	// Set log file to default if log-to-file was specified
   489  	if params.LogToFile {
   490  		if params.LogFileConfig.Path != "" {
   491  			return nil, fmt.Errorf(
   492  				"log-to-file and log-file flags can't be specified together")
   493  		}
   494  		params.LogFileConfig.Path = defaultLogPath
   495  	}
   496  
   497  	if params.LogFileConfig.Path != "" {
   498  		err = logger.SetLogFileConfig(&params.LogFileConfig, nil)
   499  	}
   500  	log := logger.New(prefix)
   501  
   502  	log.Configure("", params.Debug, "")
   503  	log.Info("KBFS version %s", VersionString())
   504  
   505  	if err != nil {
   506  		log.Warning("Failed to setup log file %q: %+v",
   507  			params.LogFileConfig.Path, err)
   508  	}
   509  
   510  	return log, err
   511  }
   512  
   513  // InitLog sets up logging switching to a log file if necessary.
   514  // Returns a valid logger even on error, which are non-fatal, thus
   515  // errors from this function may be ignored.
   516  // Possible errors are logged to the logger returned.
   517  func InitLog(params InitParams, ctx Context) (logger.Logger, error) {
   518  	return InitLogWithPrefix(params, ctx, "kbfs", defaultLogPath(ctx))
   519  }
   520  
   521  // InitWithLogPrefix initializes a config and returns it, given a prefix.
   522  //
   523  // onInterruptFn is called whenever an interrupt signal is received
   524  // (e.g., if the user hits Ctrl-C).
   525  //
   526  // Init should be called at the beginning of main. Shutdown (see
   527  // below) should then be called at the end of main (usually via
   528  // defer).
   529  //
   530  // The keybaseServiceCn argument is to specify a custom service and
   531  // crypto (for non-RPC environments) like mobile. If this is nil, we'll
   532  // use the default RPC implementation.
   533  func InitWithLogPrefix(
   534  	ctx context.Context, kbCtx Context, params InitParams,
   535  	keybaseServiceCn KeybaseServiceCn, onInterruptFn func() error,
   536  	log logger.Logger, logPrefix string) (cfg Config, err error) {
   537  	done := make(chan struct{})
   538  	interruptChan := make(chan os.Signal, 1)
   539  
   540  	SIGINT := os.Interrupt
   541  	signal.Notify(interruptChan, SIGINT, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGABRT)
   542  	if SIGPWR != NonexistentSignal {
   543  		signal.Notify(interruptChan, SIGPWR)
   544  	}
   545  
   546  	var initInterruptSignal os.Signal
   547  	var interruptErr error
   548  	go func() {
   549  		closed := false
   550  		for sig := range interruptChan {
   551  			initInterruptSignal = sig
   552  
   553  			if !closed {
   554  				close(done)
   555  				closed = true
   556  			}
   557  
   558  			// Restore stacktraces of signals that are supposed to print them
   559  			// https://golang.org/pkg/os/signal/#hdr-Default_behavior_of_signals_in_Go_programs
   560  			switch sig {
   561  			case syscall.SIGQUIT, syscall.SIGILL, syscall.SIGTRAP, syscall.SIGABRT:
   562  				_ = pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
   563  			}
   564  
   565  			if onInterruptFn != nil {
   566  				interruptErr = onInterruptFn()
   567  			}
   568  
   569  			// Continue loop only on SIGINT; exit immediately on other codes
   570  			// even if unmount has failed.
   571  			switch sig {
   572  			case SIGINT:
   573  			default:
   574  				if interruptErr != nil {
   575  					log.Info("Failed to unmount before exit: %s", interruptErr)
   576  					os.Exit(1)
   577  				} else {
   578  					// Do not return 128 + signal since kbfsfuse is not a shell command
   579  					os.Exit(0)
   580  				}
   581  			}
   582  		}
   583  	}()
   584  
   585  	// Spawn a new goroutine for `doInit` so that we can `select` on
   586  	// `done` and `errCh` below. This is particularly for the
   587  	// situation where a SIGINT comes in while `doInit` is still not
   588  	// finished (because e.g. service daemon is not up), where the
   589  	// process can fail to exit while being stuck in `doInit`.  This
   590  	// allows us to not call `os.Exit()` in the interrupt handler.
   591  	errCh := make(chan error)
   592  	go func() {
   593  		var er error
   594  		cfg, er = doInit(ctx, kbCtx, params, keybaseServiceCn, log, logPrefix)
   595  		errCh <- er
   596  	}()
   597  
   598  	select {
   599  	case <-done:
   600  		return nil, fmt.Errorf("kbfsfuse received signal=<%s>", initInterruptSignal)
   601  	case err = <-errCh:
   602  		return cfg, err
   603  	}
   604  }
   605  
   606  // Init initializes a config and returns it.
   607  //
   608  // onInterruptFn is called whenever an interrupt signal is received
   609  // (e.g., if the user hits Ctrl-C).
   610  //
   611  // Init should be called at the beginning of main. Shutdown (see
   612  // below) should then be called at the end of main (usually via
   613  // defer).
   614  //
   615  // The keybaseServiceCn argument is to specify a custom service and
   616  // crypto (for non-RPC environments) like mobile. If this is nil, we'll
   617  // use the default RPC implementation.
   618  func Init(
   619  	ctx context.Context, kbCtx Context, params InitParams,
   620  	keybaseServiceCn KeybaseServiceCn, onInterruptFn func() error,
   621  	log logger.Logger) (cfg Config, err error) {
   622  	return InitWithLogPrefix(
   623  		ctx, kbCtx, params, keybaseServiceCn, onInterruptFn, log, "kbfs")
   624  }
   625  
   626  func getInitMode(
   627  	ctx context.Context, kbCtx Context, params InitParams,
   628  	log logger.Logger) (InitMode, error) {
   629  	mode := InitDefault
   630  
   631  	// Use the KBFS mode from the config file if none is provided on
   632  	// the command line.
   633  	modeString := params.Mode
   634  	if modeString == "" {
   635  		config := kbCtx.GetEnv().GetConfig()
   636  		configModeString, ok := config.GetStringAtPath(configModeStr)
   637  		if ok {
   638  			log.CDebugf(
   639  				ctx, "Using mode from config file: %s", configModeString)
   640  			modeString = configModeString
   641  		} else {
   642  			modeString = InitDefaultString
   643  		}
   644  	}
   645  
   646  	switch modeString {
   647  	case InitDefaultString:
   648  		// Already the default
   649  	case InitMinimalString:
   650  		mode = InitMinimal
   651  	case InitSingleOpString:
   652  		mode = InitSingleOp
   653  	case InitConstrainedString:
   654  		mode = InitConstrained
   655  	case InitMemoryLimitedString:
   656  		mode = InitMemoryLimited
   657  	case InitTestSearchString:
   658  		mode = InitTestSearch
   659  	case InitSingleOpWithQRString:
   660  		mode = InitSingleOpWithQR
   661  	default:
   662  		return nil, fmt.Errorf("Unexpected mode: %s", params.Mode)
   663  	}
   664  
   665  	log.CDebugf(ctx, "Initializing in %s mode", modeString)
   666  	kbCtx.GetPerfLog().CDebugf(
   667  		ctx, "KBFS initializing in %s mode, version %s", modeString,
   668  		VersionString())
   669  
   670  	return NewInitModeFromType(mode), nil
   671  }
   672  
   673  func getCleanBlockCacheCapacity(
   674  	ctx context.Context, kbCtx Context, params InitParams,
   675  	log logger.Logger) uint64 {
   676  	cap := params.CleanBlockCacheCapacity
   677  
   678  	// Use the capacity from the config file if none is provided on
   679  	// the command line.
   680  	if cap == 0 {
   681  		config := kbCtx.GetEnv().GetConfig()
   682  		capInt, ok := config.GetIntAtPath(configBlockCacheMemMaxBytesStr)
   683  		if ok {
   684  			log.CDebugf(
   685  				ctx, "Using block cache capacity from config file: %d", capInt)
   686  			cap = uint64(capInt)
   687  		}
   688  	}
   689  
   690  	return cap
   691  }
   692  
   693  func getCacheFrac(
   694  	ctx context.Context, kbCtx Context, param, defaultVal float64,
   695  	configKey string, log logger.Logger) float64 {
   696  	frac := param
   697  
   698  	// Use the fraction from the config file if none is provided on
   699  	// the command line.
   700  	if frac == 0 {
   701  		config := kbCtx.GetEnv().GetConfig()
   702  		configFrac, ok := config.GetFloatAtPath(configKey)
   703  		if ok {
   704  			log.CDebugf(
   705  				ctx, "Using %s value from config file: %f", configKey,
   706  				configFrac)
   707  			frac = configFrac
   708  		}
   709  	}
   710  
   711  	if frac == 0 {
   712  		frac = defaultVal
   713  	}
   714  
   715  	return frac
   716  }
   717  
   718  func doInit(
   719  	ctx context.Context, kbCtx Context, params InitParams,
   720  	keybaseServiceCn KeybaseServiceCn, log logger.Logger,
   721  	logPrefix string) (Config, error) {
   722  	ctx = CtxWithRandomIDReplayable(ctx, CtxInitKey, CtxInitID, log)
   723  
   724  	initMode, err := getInitMode(ctx, kbCtx, params, log)
   725  	if err != nil {
   726  		return nil, err
   727  	}
   728  
   729  	config := NewConfigLocal(initMode,
   730  		func(module string) logger.Logger {
   731  			mname := logPrefix
   732  			if module != "" {
   733  				mname += fmt.Sprintf("(%s)", module)
   734  			}
   735  			lg := logger.New(mname)
   736  			if params.Debug {
   737  				// Turn on debugging.  TODO: allow a proper log file and
   738  				// style to be specified.
   739  				lg.Configure("", true, "")
   740  			}
   741  			return lg
   742  		}, params.StorageRoot, params.DiskCacheMode, kbCtx)
   743  	config.SetVLogLevel(kbCtx.GetVDebugSetting())
   744  
   745  	if cap := getCleanBlockCacheCapacity(ctx, kbCtx, params, log); cap > 0 {
   746  		log.CDebugf(
   747  			ctx, "Overriding default clean block cache capacity from %d to %d",
   748  			config.BlockCache().GetCleanBytesCapacity(), cap)
   749  		config.BlockCache().SetCleanBytesCapacity(cap)
   750  	}
   751  
   752  	workers := config.Mode().BlockWorkers()
   753  	prefetchWorkers := config.Mode().PrefetchWorkers()
   754  	throttledPrefetchPeriod := config.Mode().ThrottledPrefetchPeriod()
   755  	config.SetBlockOps(NewBlockOpsStandard(
   756  		config, workers, prefetchWorkers, throttledPrefetchPeriod, kbCtx))
   757  
   758  	bsplitter, err := data.NewBlockSplitterSimple(
   759  		data.MaxBlockSizeBytesDefault, 8*1024, config.Codec())
   760  	if err != nil {
   761  		return nil, err
   762  	}
   763  	err = bsplitter.SetMaxDirEntriesByBlockSize(config.Codec())
   764  	if err != nil {
   765  		return nil, err
   766  	}
   767  	config.SetBlockSplitter(bsplitter)
   768  
   769  	if registry := config.MetricsRegistry(); registry != nil {
   770  		keyCache := config.KeyCache()
   771  		keyCache = NewKeyCacheMeasured(keyCache, registry)
   772  		config.SetKeyCache(keyCache)
   773  
   774  		keyBundleCache := config.KeyBundleCache()
   775  		keyBundleCache = libkey.NewKeyBundleCacheMeasured(
   776  			keyBundleCache, registry)
   777  		config.SetKeyBundleCache(keyBundleCache)
   778  	}
   779  
   780  	config.SetMetadataVersion(params.MetadataVersion)
   781  	config.SetBlockCryptVersion(params.BlockCryptVersion)
   782  	config.SetTLFValidDuration(params.TLFValidDuration)
   783  	config.SetBGFlushPeriod(params.BGFlushPeriod)
   784  
   785  	kbfsLog := config.MakeLogger("")
   786  
   787  	// Initialize Keybase service connection. This needs to happen before
   788  	// KBPKI client.
   789  	if keybaseServiceCn == nil {
   790  		keybaseServiceCn = keybaseDaemon{}
   791  	}
   792  	service, err := keybaseServiceCn.NewKeybaseService(
   793  		config, params, kbCtx, kbfsLog)
   794  	if err != nil {
   795  		return nil, fmt.Errorf("problem creating service: %s", err)
   796  	}
   797  	if registry := config.MetricsRegistry(); registry != nil {
   798  		service = NewKeybaseServiceMeasured(service, registry)
   799  	}
   800  	config.SetKeybaseService(service)
   801  
   802  	// Initialize KBPKI client (needed for KBFSOps, MD Server, and Chat).
   803  	k := NewKBPKIClient(config, kbfsLog)
   804  	config.SetKBPKI(k)
   805  
   806  	// Initialize Chat client (for file edit notifications).
   807  	chat, err := keybaseServiceCn.NewChat(config, params, kbCtx, kbfsLog)
   808  	if err != nil {
   809  		return nil, fmt.Errorf("problem creating chat: %s", err)
   810  	}
   811  	config.SetChat(chat)
   812  
   813  	initDoneCh := make(chan struct{})
   814  	kbfsOps := NewKBFSOpsStandard(kbCtx, config, initDoneCh)
   815  	defer close(initDoneCh)
   816  	config.SetKBFSOps(kbfsOps)
   817  	config.SetNotifier(kbfsOps)
   818  	config.SetKeyManager(NewKeyManagerStandard(config))
   819  	config.SetMDOps(NewMDOpsStandard(config))
   820  
   821  	config.SetDiskBlockCacheFraction(getCacheFrac(
   822  		ctx, kbCtx, params.DiskBlockCacheFraction,
   823  		defaultDiskBlockCacheFraction, configBlockCacheDiskMaxFracStr, log))
   824  	config.SetSyncBlockCacheFraction(getCacheFrac(
   825  		ctx, kbCtx, params.SyncBlockCacheFraction,
   826  		defaultSyncBlockCacheFraction, configBlockCacheSyncMaxFracStr, log))
   827  	err = config.EnableDiskLimiter(params.StorageRoot)
   828  	if err != nil {
   829  		log.CWarningf(ctx, "Could not enable disk limiter: %+v", err)
   830  		return nil, err
   831  	}
   832  
   833  	kbfsOps.favs.Initialize(ctx)
   834  
   835  	config.SetReporter(NewReporterKBPKI(config, 10, 1000))
   836  
   837  	// Initialize Crypto client (needed for MD and Block servers).
   838  	crypto, err := keybaseServiceCn.NewCrypto(config, params, kbCtx, kbfsLog)
   839  	if err != nil {
   840  		return nil, fmt.Errorf("problem creating crypto: %s", err)
   841  	}
   842  	config.SetCrypto(crypto)
   843  
   844  	// Initialize MDServer connection.
   845  	mdServer, err := makeMDServer(kbCtx, config, params.MDServerAddr, kbCtx.NewRPCLogFactory(), log)
   846  	if err != nil {
   847  		return nil, fmt.Errorf("problem creating MD server: %+v", err)
   848  	}
   849  	config.SetMDServer(mdServer)
   850  
   851  	// Must do this after setting the md server, since it depends on
   852  	// being able to fetch MDs.
   853  	go kbfsOps.initSyncedTlfs()
   854  
   855  	// Initialize KeyServer connection.  MDServer is the KeyServer at the
   856  	// moment.
   857  	keyServer, err := makeKeyServer(config, params.MDServerAddr, log)
   858  	if err != nil {
   859  		return nil, fmt.Errorf("problem creating key server: %+v", err)
   860  	}
   861  	if registry := config.MetricsRegistry(); registry != nil {
   862  		keyServer = libkey.NewKeyServerMeasured(keyServer, registry)
   863  	}
   864  	config.SetKeyServer(keyServer)
   865  
   866  	// Initialize BlockServer connection.
   867  	bserv, err := makeBlockServer(
   868  		kbCtx, config, params.BServerAddr, kbCtx.NewRPCLogFactory(), log)
   869  	if err != nil {
   870  		return nil, fmt.Errorf("cannot open block database: %+v", err)
   871  	}
   872  	if registry := config.MetricsRegistry(); registry != nil {
   873  		bserv = NewBlockServerMeasured(bserv, registry)
   874  	}
   875  	config.SetBlockServer(bserv)
   876  
   877  	// Don't trigger cache initialization warnings in single-op mode
   878  	// (e.g., during a git pull), because KBFS might be running in
   879  	// constrained mode and the cache remote proxy wouldn't be
   880  	// available (which is fine).
   881  	doSendWarnings := !config.Mode().IsSingleOp()
   882  	err = config.MakeDiskBlockCacheIfNotExists()
   883  	if err != nil {
   884  		log.CWarningf(ctx, "Could not initialize disk cache: %+v", err)
   885  		config.GetPerfLog().CDebugf(
   886  			ctx, "KBFS could not initialize disk cache: %v", err)
   887  		notification := &keybase1.FSNotification{
   888  			StatusCode:       keybase1.FSStatusCode_ERROR,
   889  			NotificationType: keybase1.FSNotificationType_INITIALIZED,
   890  			ErrorType:        keybase1.FSErrorType_DISK_CACHE_ERROR_LOG_SEND,
   891  		}
   892  		if doSendWarnings {
   893  			defer config.Reporter().Notify(ctx, notification)
   894  		}
   895  	} else {
   896  		log.CDebugf(ctx, "Disk cache of type \"%s\" enabled",
   897  			params.DiskCacheMode.String())
   898  	}
   899  
   900  	err = config.MakeDiskMDCacheIfNotExists()
   901  	if err != nil {
   902  		log.CWarningf(ctx, "Could not initialize MD cache: %+v", err)
   903  		notification := &keybase1.FSNotification{
   904  			StatusCode:       keybase1.FSStatusCode_ERROR,
   905  			NotificationType: keybase1.FSNotificationType_INITIALIZED,
   906  			ErrorType:        keybase1.FSErrorType_DISK_CACHE_ERROR_LOG_SEND,
   907  		}
   908  		if doSendWarnings {
   909  			defer config.Reporter().Notify(ctx, notification)
   910  		}
   911  	} else {
   912  		log.CDebugf(ctx, "Disk MD cache enabled")
   913  	}
   914  
   915  	err = config.MakeDiskQuotaCacheIfNotExists()
   916  	if err != nil {
   917  		log.CWarningf(ctx, "Could not initialize disk quota cache: %+v", err)
   918  		notification := &keybase1.FSNotification{
   919  			StatusCode:       keybase1.FSStatusCode_ERROR,
   920  			NotificationType: keybase1.FSNotificationType_INITIALIZED,
   921  			ErrorType:        keybase1.FSErrorType_DISK_CACHE_ERROR_LOG_SEND,
   922  		}
   923  		if doSendWarnings {
   924  			defer config.Reporter().Notify(ctx, notification)
   925  		}
   926  	} else {
   927  		log.CDebugf(ctx, "Disk quota cache enabled")
   928  	}
   929  
   930  	err = config.MakeBlockMetadataStoreIfNotExists()
   931  	if err != nil {
   932  		log.CWarningf(ctx,
   933  			"Could not initialize block metadata store: %+v", err)
   934  		/*
   935  			notification := &keybase1.FSNotification{
   936  				StatusCode:       keybase1.FSStatusCode_ERROR,
   937  				NotificationType: keybase1.FSNotificationType_INITIALIZED,
   938  				ErrorType:        keybase1.FSErrorType_DISK_CACHE_ERROR_LOG_SEND,
   939  			}
   940  			defer config.Reporter().Notify(ctx, notification)
   941  		*/
   942  	}
   943  	log.CDebugf(ctx, "Disk block metadata store cache enabled")
   944  
   945  	if config.Mode().KBFSServiceEnabled() {
   946  		// Initialize kbfsService only when we run a full KBFS process.
   947  		// This requires the disk block cache to have been initialized, if it
   948  		// should be initialized.
   949  		kbfsService, err := NewKBFSService(kbCtx, config)
   950  		if err != nil {
   951  			// This error shouldn't be fatal
   952  			log.CWarningf(ctx, "Error starting RPC server for KBFS: %+v", err)
   953  		} else {
   954  			config.SetKBFSService(kbfsService)
   955  			log.CDebugf(ctx, "Started RPC server for KBFS")
   956  		}
   957  	}
   958  
   959  	ctx60s, cancel := context.WithTimeout(ctx, 60*time.Second)
   960  	defer cancel()
   961  	// TODO: Don't turn on journaling if either -bserver or
   962  	// -mdserver point to local implementations.
   963  	if params.EnableJournal && config.Mode().JournalEnabled() {
   964  		journalRoot := filepath.Join(params.StorageRoot, "kbfs_journal")
   965  		err = config.EnableJournaling(ctx60s, journalRoot,
   966  			params.TLFJournalBackgroundWorkStatus)
   967  		if err != nil {
   968  			log.CWarningf(ctx, "Could not initialize journal server: %+v", err)
   969  		}
   970  		log.CDebugf(ctx, "Journaling enabled")
   971  	}
   972  
   973  	if params.BGFlushDirOpBatchSize < 1 {
   974  		return nil, fmt.Errorf(
   975  			"Illegal sync batch size: %d", params.BGFlushDirOpBatchSize)
   976  	}
   977  	log.CDebugf(ctx, "Enabling a dir op batch size of %d",
   978  		params.BGFlushDirOpBatchSize)
   979  	config.SetBGFlushDirOpBatchSize(params.BGFlushDirOpBatchSize)
   980  
   981  	if config.Mode().OldStorageRootCleaningEnabled() {
   982  		go cleanOldTempStorageRoots(config)
   983  	}
   984  
   985  	return config, nil
   986  }
   987  
   988  // Shutdown does any necessary shutdown tasks for libkbfs. Shutdown
   989  // should be called at the end of main.
   990  func Shutdown() {}