github.com/keybase/client/go@v0.0.0-20241007131713-f10651d043c8/kbfs/libkbfs/init.go (about) 1 // Copyright 2016 Keybase Inc. All rights reserved. 2 // Use of this source code is governed by a BSD 3 // license that can be found in the LICENSE file. 4 5 package libkbfs 6 7 import ( 8 "context" 9 "errors" 10 "flag" 11 "fmt" 12 "os" 13 "os/signal" 14 "path/filepath" 15 "runtime/pprof" 16 "strings" 17 "syscall" 18 "time" 19 20 "github.com/keybase/client/go/kbconst" 21 "github.com/keybase/client/go/kbfs/data" 22 "github.com/keybase/client/go/kbfs/kbfscrypto" 23 "github.com/keybase/client/go/kbfs/kbfsmd" 24 "github.com/keybase/client/go/kbfs/libkey" 25 "github.com/keybase/client/go/libkb" 26 "github.com/keybase/client/go/logger" 27 "github.com/keybase/client/go/protocol/keybase1" 28 "github.com/keybase/go-framed-msgpack-rpc/rpc" 29 ) 30 31 const ( 32 // InitDefaultString is the normal mode for when KBFS data will be 33 // read and written. 34 InitDefaultString string = "default" 35 // InitMinimalString is for when KBFS will only be used as a MD 36 // lookup layer (e.g., for chat on mobile). 37 InitMinimalString = "minimal" 38 // InitSingleOpString is for when KBFS will only be used for a 39 // single logical operation (e.g., as a git remote helper). 40 InitSingleOpString = "singleOp" 41 // InitConstrainedString is for when KBFS will use constrained 42 // resources. 43 InitConstrainedString = "constrained" 44 // InitMemoryLimitedString is for when KBFS will use memory limited 45 // resources. 46 InitMemoryLimitedString = "memoryLimited" 47 // InitTestSearchString is for when KBFS will index synced TLFs. 48 InitTestSearchString = "testSearch" 49 // InitSingleOpWithQRString is for when KBFS will only be used for 50 // a single logical operation (e.g., as a git remote helper), with 51 // QR enabled. 52 InitSingleOpWithQRString = "singleOpQR" 53 ) 54 55 // CtxInitTagKey is the type used for unique context tags for KBFS init. 56 type CtxInitTagKey int 57 58 const ( 59 // CtxInitKey is the type of the tag for unique operation IDs 60 // for KBFS init. 61 CtxInitKey CtxInitTagKey = iota 62 ) 63 64 // CtxInitID is the display name for the unique operation 65 // init ID tag. 66 const CtxInitID = "KBFSINIT" 67 68 // AdditionalProtocolCreator creates an additional protocol. 69 type AdditionalProtocolCreator func(Context, Config) (rpc.Protocol, error) 70 71 const ( 72 configModeStr = "kbfs.mode" 73 configBlockCacheMemMaxBytesStr = "kbfs.block_cache.mem_max_bytes" 74 configBlockCacheDiskMaxFracStr = "kbfs.block_cache.disk_max_fraction" 75 configBlockCacheSyncMaxFracStr = "kbfs.block_cache.sync_max_fraction" 76 ) 77 78 // InitParams contains the initialization parameters for Init(). It is 79 // usually filled in by the flags parser passed into AddFlags(). 80 type InitParams struct { 81 // Whether to print debug messages. 82 Debug bool 83 84 // If non-empty, the host:port of the block server. If empty, 85 // a default value is used depending on the run mode. Can also 86 // be "memory" for an in-memory test server or 87 // "dir:/path/to/dir" for an on-disk test server. 88 BServerAddr string 89 90 // If non-empty the host:port of the metadata server. If 91 // empty, a default value is used depending on the run mode. 92 // Can also be "memory" for an in-memory test server or 93 // "dir:/path/to/dir" for an on-disk test server. 94 MDServerAddr string 95 96 // If non-zero, specifies the capacity (in bytes) of the block cache. If 97 // zero, the capacity is set using getDefaultBlockCacheCapacity(). 98 CleanBlockCacheCapacity uint64 99 100 // Fake local user name. 101 LocalUser string 102 103 // Where to put favorites. Has an effect only when LocalUser 104 // is non-empty, in which case it must be either "memory" or 105 // "dir:/path/to/dir". 106 LocalFavoriteStorage string 107 108 // TLFValidDuration is the duration that TLFs are valid 109 // before marked for lazy revalidation. 110 TLFValidDuration time.Duration 111 112 // MetadataVersion is the default version of metadata to use 113 // when creating new metadata. 114 MetadataVersion kbfsmd.MetadataVer 115 116 // BlockCryptVersion is the encryption version to use when 117 // encrypting new blocks. 118 BlockCryptVersion kbfscrypto.EncryptionVer 119 120 // LogToFile if true, logs to a default file location. 121 LogToFile bool 122 123 // LogFileConfig tells us where to log and rotation config. 124 LogFileConfig logger.LogFileConfig 125 126 // TLFJournalBackgroundWorkStatus is the status to use to 127 // pass into JournalManager.enableJournaling. Only has an effect when 128 // EnableJournal is non-empty. 129 TLFJournalBackgroundWorkStatus TLFJournalBackgroundWorkStatus 130 131 // AdditionalProtocolCreators are for adding additional protocols that we 132 // should handle for service to call in. 133 AdditionalProtocolCreators []AdditionalProtocolCreator 134 135 // EnableJournal enables journaling. 136 EnableJournal bool 137 138 // DiskCacheMode specifies which mode to start the disk cache. 139 DiskCacheMode DiskCacheMode 140 141 // StorageRoot, if non-empty, points to a local directory to put its local 142 // databases for things like the journal or disk cache. 143 StorageRoot string 144 145 // BGFlushPeriod indicates how long to wait for a batch to fill up 146 // before syncing a set of changes on a TLF to the servers. 147 BGFlushPeriod time.Duration 148 149 // BGFlushDirOpBatchSize indicates how many directory operations 150 // in a TLF should be batched together in a single background 151 // flush. 152 BGFlushDirOpBatchSize int 153 154 // Mode describes how KBFS should initialize itself. 155 Mode string 156 157 // DiskBlockCacheFraction indicates what fraction of free space on the disk 158 // is allowed to be occupied by the KBFS disk block cache. 159 DiskBlockCacheFraction float64 160 161 // SyncBlockCacheFraction indicates what fraction of free space on the disk 162 // is allowed to be occupied by the KBFS sync block cache for offline use. 163 SyncBlockCacheFraction float64 164 } 165 166 // defaultBServer returns the default value for the -bserver flag. 167 func defaultBServer(ctx Context) string { 168 switch ctx.GetRunMode() { 169 case kbconst.DevelRunMode: 170 return memoryAddr 171 case kbconst.StagingRunMode: 172 return ` 173 bserver-0.dev.keybase.io:443,bserver-1.dev.keybase.io:443` 174 case kbconst.ProductionRunMode: 175 return ` 176 bserver-0.kbfs.keybaseapi.com:443,bserver-1.kbfs.keybaseapi.com:443; 177 bserver-0.kbfs.keybase.io:443,bserver-1.kbfs.keybase.io:443` 178 default: 179 return "" 180 } 181 } 182 183 // defaultMDServer returns the default value for the -mdserver flag. 184 func defaultMDServer(ctx Context) string { 185 switch ctx.GetRunMode() { 186 case kbconst.DevelRunMode: 187 return memoryAddr 188 case kbconst.StagingRunMode: 189 return ` 190 mdserver-0.dev.keybase.io:443,mdserver-1.dev.keybase.io:443` 191 case kbconst.ProductionRunMode: 192 return ` 193 mdserver-0.kbfs.keybaseapi.com:443,mdserver-1.kbfs.keybaseapi.com:443; 194 mdserver-0.kbfs.keybase.io:443,mdserver-1.kbfs.keybase.io:443` 195 default: 196 return "" 197 } 198 } 199 200 // defaultMetadataVersion returns the default metadata version per run mode. 201 func defaultMetadataVersion(ctx Context) kbfsmd.MetadataVer { 202 switch ctx.GetRunMode() { 203 case kbconst.DevelRunMode: 204 return kbfsmd.ImplicitTeamsVer 205 case kbconst.StagingRunMode: 206 return kbfsmd.ImplicitTeamsVer 207 case kbconst.ProductionRunMode: 208 return kbfsmd.ImplicitTeamsVer 209 default: 210 return kbfsmd.ImplicitTeamsVer 211 } 212 } 213 214 func defaultLogPath(ctx Context) string { 215 return filepath.Join(ctx.GetLogDir(), libkb.KBFSLogFileName) 216 } 217 218 // DefaultInitParams returns default init params 219 func DefaultInitParams(ctx Context) InitParams { 220 journalEnv := os.Getenv("KBFS_DEFAULT_ENABLE_JOURNAL_VALUE") 221 if journalEnv == "" { 222 journalEnv = "true" 223 } 224 return InitParams{ 225 Debug: BoolForString(os.Getenv("KBFS_DEBUG")), 226 BServerAddr: defaultBServer(ctx), 227 MDServerAddr: defaultMDServer(ctx), 228 TLFValidDuration: tlfValidDurationDefault, 229 MetadataVersion: defaultMetadataVersion(ctx), 230 BlockCryptVersion: kbfscrypto.EncryptionSecretboxWithKeyNonce, 231 LogFileConfig: logger.LogFileConfig{ 232 MaxAge: 30 * 24 * time.Hour, 233 MaxSize: 128 * 1024 * 1024, 234 MaxKeepFiles: 3, 235 }, 236 TLFJournalBackgroundWorkStatus: TLFJournalBackgroundWorkEnabled, 237 StorageRoot: ctx.GetDataDir(), 238 BGFlushPeriod: bgFlushPeriodDefault, 239 BGFlushDirOpBatchSize: bgFlushDirOpBatchSizeDefault, 240 EnableJournal: BoolForString(journalEnv), 241 DiskCacheMode: DiskCacheModeLocal, 242 Mode: "", 243 } 244 } 245 246 // AddFlagsWithDefaults adds libkbfs flags to the given FlagSet, given 247 // a set of default flags. Returns an InitParams that will be filled 248 // in once the given FlagSet is parsed. 249 func AddFlagsWithDefaults( 250 flags *flag.FlagSet, defaultParams InitParams, 251 defaultLogPath string) *InitParams { 252 var params InitParams 253 flags.BoolVar(¶ms.Debug, "debug", defaultParams.Debug, 254 "Print debug messages") 255 256 flags.StringVar(¶ms.BServerAddr, "bserver", defaultParams.BServerAddr, 257 "host:port of the block server, 'memory', or 'dir:/path/to/dir'") 258 flags.StringVar(¶ms.MDServerAddr, "mdserver", 259 defaultParams.MDServerAddr, 260 "host:port of the metadata server, 'memory', or 'dir:/path/to/dir'") 261 flags.StringVar(¶ms.LocalUser, "localuser", defaultParams.LocalUser, 262 "fake local user") 263 flags.StringVar(¶ms.LocalFavoriteStorage, "local-fav-storage", 264 defaultParams.LocalFavoriteStorage, 265 "where to put favorites; used only when -localuser is set, then must "+ 266 "either be 'memory' or 'dir:/path/to/dir'") 267 flags.DurationVar(¶ms.TLFValidDuration, "tlf-valid", 268 defaultParams.TLFValidDuration, 269 "time tlfs are valid before redoing identification") 270 flags.BoolVar(¶ms.LogToFile, "log-to-file", defaultParams.LogToFile, 271 fmt.Sprintf("Log to default file: %s", defaultLogPath)) 272 flags.StringVar(¶ms.LogFileConfig.Path, "log-file", "", 273 "Path to log file") 274 flags.DurationVar(¶ms.LogFileConfig.MaxAge, "log-file-max-age", 275 defaultParams.LogFileConfig.MaxAge, 276 "Maximum age of a log file before rotation") 277 params.LogFileConfig.MaxSize = defaultParams.LogFileConfig.MaxSize 278 flags.Var(SizeFlag{¶ms.LogFileConfig.MaxSize}, "log-file-max-size", 279 "Maximum size of a log file before rotation") 280 // The default is to *DELETE* old log files for kbfs. 281 flags.IntVar(¶ms.LogFileConfig.MaxKeepFiles, "log-file-max-keep-files", 282 defaultParams.LogFileConfig.MaxKeepFiles, "Maximum number of log "+ 283 "files for this service, older ones are deleted. 0 for infinite.") 284 flags.Uint64Var(¶ms.CleanBlockCacheCapacity, "clean-bcache-cap", 285 defaultParams.CleanBlockCacheCapacity, 286 "If non-zero, specify the capacity of clean block cache. If zero, "+ 287 "the capacity is set based on system RAM.") 288 flags.StringVar(¶ms.StorageRoot, "storage-root", 289 defaultParams.StorageRoot, "Specifies where Keybase will store its "+ 290 "local databases for the journal and disk cache.") 291 params.DiskCacheMode = defaultParams.DiskCacheMode 292 flags.Var(¶ms.DiskCacheMode, "disk-cache-mode", 293 "Sets the mode for the disk cache. If 'local', then it uses a "+ 294 "subdirectory of -storage-root to store the cache. If 'remote', "+ 295 "then it connects to the local KBFS instance and delegates disk "+ 296 "cache operations to it.") 297 flags.BoolVar(¶ms.EnableJournal, "enable-journal", 298 defaultParams.EnableJournal, "Enables write journaling for TLFs.") 299 300 // No real need to enable setting 301 // params.TLFJournalBackgroundWorkStatus via a flag. 302 params.TLFJournalBackgroundWorkStatus = 303 defaultParams.TLFJournalBackgroundWorkStatus 304 305 flags.DurationVar(¶ms.BGFlushPeriod, "sync-batch-period", 306 defaultParams.BGFlushPeriod, 307 "The amount of time to wait before syncing data in a TLF, if the "+ 308 "batch size doesn't fill up.") 309 flags.IntVar(¶ms.BGFlushDirOpBatchSize, "sync-batch-size", 310 defaultParams.BGFlushDirOpBatchSize, 311 "The number of unflushed directory operations in a TLF that will "+ 312 "trigger an immediate data sync.") 313 314 flags.IntVar((*int)(¶ms.MetadataVersion), "md-version", 315 int(defaultParams.MetadataVersion), 316 "Metadata version to use when creating new metadata") 317 flags.IntVar((*int)(¶ms.BlockCryptVersion), "block-crypt-version", 318 int(defaultParams.BlockCryptVersion), 319 "Encryption version to use when encrypting new blocks") 320 flags.StringVar(¶ms.Mode, "mode", defaultParams.Mode, 321 fmt.Sprintf("Overall initialization mode for KBFS, indicating how "+ 322 "heavy-weight it can be (%s, %s, %s, %s or %s)", InitDefaultString, 323 InitMinimalString, InitSingleOpString, InitConstrainedString, 324 InitMemoryLimitedString)) 325 326 flags.Float64Var(¶ms.DiskBlockCacheFraction, 327 "disk-block-cache-fraction", defaultParams.DiskBlockCacheFraction, 328 "The portion of the free disk space that KBFS will use for caching ") 329 330 flags.Float64Var(¶ms.SyncBlockCacheFraction, 331 "sync-block-cache-fraction", defaultParams.SyncBlockCacheFraction, 332 "The portion of the free disk space that KBFS will use for offline storage") 333 334 return ¶ms 335 } 336 337 // AddFlags adds libkbfs flags to the given FlagSet. Returns an 338 // InitParams that will be filled in once the given FlagSet is parsed. 339 func AddFlags(flags *flag.FlagSet, ctx Context) *InitParams { 340 return AddFlagsWithDefaults( 341 flags, DefaultInitParams(ctx), defaultLogPath(ctx)) 342 } 343 344 // GetRemoteUsageString returns a string describing the flags to use 345 // to run against remote KBFS servers. 346 func GetRemoteUsageString() string { 347 return ` [-debug] 348 [-bserver=host:port] [-mdserver=host:port] 349 [-log-to-file] [-log-file=path/to/file] [-clean-bcache-cap=0]` 350 } 351 352 // GetLocalUsageString returns a string describing the flags to use to 353 // run in a local testing environment. 354 func GetLocalUsageString() string { 355 return ` [-debug] 356 [-bserver=(memory | dir:/path/to/dir | host:port)] 357 [-mdserver=(memory | dir:/path/to/dir | host:port)] 358 [-localuser=<user>] 359 [-local-fav-storage=(memory | dir:/path/to/dir)] 360 [-log-to-file] [-log-file=path/to/file] [-clean-bcache-cap=0]` 361 } 362 363 // GetDefaultsUsageString returns a string describing the default 364 // values of flags based on the run mode. 365 func GetDefaultsUsageString(ctx Context) string { 366 runMode := ctx.GetRunMode() 367 defaultBServer := defaultBServer(ctx) 368 defaultMDServer := defaultMDServer(ctx) 369 return fmt.Sprintf(` (KEYBASE_RUN_MODE=%s) 370 -bserver=%s 371 -mdserver=%s`, 372 runMode, defaultBServer, defaultMDServer) 373 } 374 375 const memoryAddr = "memory" 376 377 const dirAddrPrefix = "dir:" 378 379 func parseRootDir(addr string) (string, bool) { 380 if !strings.HasPrefix(addr, dirAddrPrefix) { 381 return "", false 382 } 383 serverRootDir := addr[len(dirAddrPrefix):] 384 if len(serverRootDir) == 0 { 385 return "", false 386 } 387 return serverRootDir, true 388 } 389 390 func makeMDServer(kbCtx Context, config Config, mdserverAddr string, 391 rpcLogFactory rpc.LogFactory, log logger.Logger) ( 392 MDServer, error) { 393 if mdserverAddr == memoryAddr { 394 log.Debug("Using in-memory mdserver") 395 // local in-memory MD server 396 return NewMDServerMemory(mdServerLocalConfigAdapter{config}) 397 } 398 399 if len(mdserverAddr) == 0 { 400 return nil, errors.New("Empty MD server address") 401 } 402 403 if serverRootDir, ok := parseRootDir(mdserverAddr); ok { 404 log.Debug("Using on-disk mdserver at %s", serverRootDir) 405 // local persistent MD server 406 return MakeDiskMDServer(config, serverRootDir) 407 } 408 409 remote, err := rpc.ParsePrioritizedRoundRobinRemote(mdserverAddr) 410 if err != nil { 411 return nil, err 412 } 413 // remote MD server. this can't fail. reconnection attempts 414 // will be automatic. 415 log.Debug("Using remote mdserver %s", remote) 416 mdServer := NewMDServerRemote(kbCtx, config, remote, rpcLogFactory) 417 return mdServer, nil 418 } 419 420 func makeKeyServer( 421 config Config, keyserverAddr string, log logger.Logger) ( 422 libkey.KeyServer, error) { 423 keyOpsConfig := keyOpsConfigWrapper{config} 424 if keyserverAddr == memoryAddr { 425 log.Debug("Using in-memory keyserver") 426 // local in-memory key server 427 return libkey.NewKeyServerMemory(keyOpsConfig, log) 428 } 429 430 if len(keyserverAddr) == 0 { 431 return nil, errors.New("Empty key server address") 432 } 433 434 if serverRootDir, ok := parseRootDir(keyserverAddr); ok { 435 log.Debug("Using on-disk keyserver at %s", serverRootDir) 436 // local persistent key server 437 keyPath := filepath.Join(serverRootDir, "kbfs_key") 438 return libkey.NewKeyServerDir(keyOpsConfig, log, keyPath) 439 } 440 441 log.Debug("Using remote keyserver %s (same as mdserver)", keyserverAddr) 442 // currently the MD server also acts as the key server. 443 keyServer, ok := config.MDServer().(libkey.KeyServer) 444 if !ok { 445 return nil, errors.New("MD server is not a key server") 446 } 447 return keyServer, nil 448 } 449 450 func makeBlockServer(kbCtx Context, config Config, bserverAddr string, 451 rpcLogFactory rpc.LogFactory, 452 log logger.Logger) (BlockServer, error) { 453 if bserverAddr == memoryAddr { 454 log.Debug("Using in-memory bserver") 455 bserverLog := config.MakeLogger("BSM") 456 // local in-memory block server 457 return NewBlockServerMemory(bserverLog), nil 458 } 459 460 if len(bserverAddr) == 0 { 461 return nil, errors.New("Empty block server address") 462 } 463 464 if serverRootDir, ok := parseRootDir(bserverAddr); ok { 465 log.Debug("Using on-disk bserver at %s", serverRootDir) 466 // local persistent block server 467 return MakeDiskBlockServer(config, serverRootDir), nil 468 } 469 470 remote, err := rpc.ParsePrioritizedRoundRobinRemote(bserverAddr) 471 if err != nil { 472 return nil, err 473 } 474 log.Debug("Using remote bserver %s", remote) 475 return NewBlockServerRemote(kbCtx, config, remote, rpcLogFactory), nil 476 } 477 478 // InitLogWithPrefix sets up logging switching to a log file if 479 // necessary, given a prefix and a default log path. Returns a valid 480 // logger even on error, which are non-fatal, thus errors from this 481 // function may be ignored. Possible errors are logged to the logger 482 // returned. 483 func InitLogWithPrefix( 484 params InitParams, ctx Context, prefix string, 485 defaultLogPath string) (logger.Logger, error) { 486 var err error 487 488 // Set log file to default if log-to-file was specified 489 if params.LogToFile { 490 if params.LogFileConfig.Path != "" { 491 return nil, fmt.Errorf( 492 "log-to-file and log-file flags can't be specified together") 493 } 494 params.LogFileConfig.Path = defaultLogPath 495 } 496 497 if params.LogFileConfig.Path != "" { 498 err = logger.SetLogFileConfig(¶ms.LogFileConfig, nil) 499 } 500 log := logger.New(prefix) 501 502 log.Configure("", params.Debug, "") 503 log.Info("KBFS version %s", VersionString()) 504 505 if err != nil { 506 log.Warning("Failed to setup log file %q: %+v", 507 params.LogFileConfig.Path, err) 508 } 509 510 return log, err 511 } 512 513 // InitLog sets up logging switching to a log file if necessary. 514 // Returns a valid logger even on error, which are non-fatal, thus 515 // errors from this function may be ignored. 516 // Possible errors are logged to the logger returned. 517 func InitLog(params InitParams, ctx Context) (logger.Logger, error) { 518 return InitLogWithPrefix(params, ctx, "kbfs", defaultLogPath(ctx)) 519 } 520 521 // InitWithLogPrefix initializes a config and returns it, given a prefix. 522 // 523 // onInterruptFn is called whenever an interrupt signal is received 524 // (e.g., if the user hits Ctrl-C). 525 // 526 // Init should be called at the beginning of main. Shutdown (see 527 // below) should then be called at the end of main (usually via 528 // defer). 529 // 530 // The keybaseServiceCn argument is to specify a custom service and 531 // crypto (for non-RPC environments) like mobile. If this is nil, we'll 532 // use the default RPC implementation. 533 func InitWithLogPrefix( 534 ctx context.Context, kbCtx Context, params InitParams, 535 keybaseServiceCn KeybaseServiceCn, onInterruptFn func() error, 536 log logger.Logger, logPrefix string) (cfg Config, err error) { 537 done := make(chan struct{}) 538 interruptChan := make(chan os.Signal, 1) 539 540 SIGINT := os.Interrupt 541 signal.Notify(interruptChan, SIGINT, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGABRT) 542 if SIGPWR != NonexistentSignal { 543 signal.Notify(interruptChan, SIGPWR) 544 } 545 546 var initInterruptSignal os.Signal 547 var interruptErr error 548 go func() { 549 closed := false 550 for sig := range interruptChan { 551 initInterruptSignal = sig 552 553 if !closed { 554 close(done) 555 closed = true 556 } 557 558 // Restore stacktraces of signals that are supposed to print them 559 // https://golang.org/pkg/os/signal/#hdr-Default_behavior_of_signals_in_Go_programs 560 switch sig { 561 case syscall.SIGQUIT, syscall.SIGILL, syscall.SIGTRAP, syscall.SIGABRT: 562 _ = pprof.Lookup("goroutine").WriteTo(os.Stdout, 1) 563 } 564 565 if onInterruptFn != nil { 566 interruptErr = onInterruptFn() 567 } 568 569 // Continue loop only on SIGINT; exit immediately on other codes 570 // even if unmount has failed. 571 switch sig { 572 case SIGINT: 573 default: 574 if interruptErr != nil { 575 log.Info("Failed to unmount before exit: %s", interruptErr) 576 os.Exit(1) 577 } else { 578 // Do not return 128 + signal since kbfsfuse is not a shell command 579 os.Exit(0) 580 } 581 } 582 } 583 }() 584 585 // Spawn a new goroutine for `doInit` so that we can `select` on 586 // `done` and `errCh` below. This is particularly for the 587 // situation where a SIGINT comes in while `doInit` is still not 588 // finished (because e.g. service daemon is not up), where the 589 // process can fail to exit while being stuck in `doInit`. This 590 // allows us to not call `os.Exit()` in the interrupt handler. 591 errCh := make(chan error) 592 go func() { 593 var er error 594 cfg, er = doInit(ctx, kbCtx, params, keybaseServiceCn, log, logPrefix) 595 errCh <- er 596 }() 597 598 select { 599 case <-done: 600 return nil, fmt.Errorf("kbfsfuse received signal=<%s>", initInterruptSignal) 601 case err = <-errCh: 602 return cfg, err 603 } 604 } 605 606 // Init initializes a config and returns it. 607 // 608 // onInterruptFn is called whenever an interrupt signal is received 609 // (e.g., if the user hits Ctrl-C). 610 // 611 // Init should be called at the beginning of main. Shutdown (see 612 // below) should then be called at the end of main (usually via 613 // defer). 614 // 615 // The keybaseServiceCn argument is to specify a custom service and 616 // crypto (for non-RPC environments) like mobile. If this is nil, we'll 617 // use the default RPC implementation. 618 func Init( 619 ctx context.Context, kbCtx Context, params InitParams, 620 keybaseServiceCn KeybaseServiceCn, onInterruptFn func() error, 621 log logger.Logger) (cfg Config, err error) { 622 return InitWithLogPrefix( 623 ctx, kbCtx, params, keybaseServiceCn, onInterruptFn, log, "kbfs") 624 } 625 626 func getInitMode( 627 ctx context.Context, kbCtx Context, params InitParams, 628 log logger.Logger) (InitMode, error) { 629 mode := InitDefault 630 631 // Use the KBFS mode from the config file if none is provided on 632 // the command line. 633 modeString := params.Mode 634 if modeString == "" { 635 config := kbCtx.GetEnv().GetConfig() 636 configModeString, ok := config.GetStringAtPath(configModeStr) 637 if ok { 638 log.CDebugf( 639 ctx, "Using mode from config file: %s", configModeString) 640 modeString = configModeString 641 } else { 642 modeString = InitDefaultString 643 } 644 } 645 646 switch modeString { 647 case InitDefaultString: 648 // Already the default 649 case InitMinimalString: 650 mode = InitMinimal 651 case InitSingleOpString: 652 mode = InitSingleOp 653 case InitConstrainedString: 654 mode = InitConstrained 655 case InitMemoryLimitedString: 656 mode = InitMemoryLimited 657 case InitTestSearchString: 658 mode = InitTestSearch 659 case InitSingleOpWithQRString: 660 mode = InitSingleOpWithQR 661 default: 662 return nil, fmt.Errorf("Unexpected mode: %s", params.Mode) 663 } 664 665 log.CDebugf(ctx, "Initializing in %s mode", modeString) 666 kbCtx.GetPerfLog().CDebugf( 667 ctx, "KBFS initializing in %s mode, version %s", modeString, 668 VersionString()) 669 670 return NewInitModeFromType(mode), nil 671 } 672 673 func getCleanBlockCacheCapacity( 674 ctx context.Context, kbCtx Context, params InitParams, 675 log logger.Logger) uint64 { 676 cap := params.CleanBlockCacheCapacity 677 678 // Use the capacity from the config file if none is provided on 679 // the command line. 680 if cap == 0 { 681 config := kbCtx.GetEnv().GetConfig() 682 capInt, ok := config.GetIntAtPath(configBlockCacheMemMaxBytesStr) 683 if ok { 684 log.CDebugf( 685 ctx, "Using block cache capacity from config file: %d", capInt) 686 cap = uint64(capInt) 687 } 688 } 689 690 return cap 691 } 692 693 func getCacheFrac( 694 ctx context.Context, kbCtx Context, param, defaultVal float64, 695 configKey string, log logger.Logger) float64 { 696 frac := param 697 698 // Use the fraction from the config file if none is provided on 699 // the command line. 700 if frac == 0 { 701 config := kbCtx.GetEnv().GetConfig() 702 configFrac, ok := config.GetFloatAtPath(configKey) 703 if ok { 704 log.CDebugf( 705 ctx, "Using %s value from config file: %f", configKey, 706 configFrac) 707 frac = configFrac 708 } 709 } 710 711 if frac == 0 { 712 frac = defaultVal 713 } 714 715 return frac 716 } 717 718 func doInit( 719 ctx context.Context, kbCtx Context, params InitParams, 720 keybaseServiceCn KeybaseServiceCn, log logger.Logger, 721 logPrefix string) (Config, error) { 722 ctx = CtxWithRandomIDReplayable(ctx, CtxInitKey, CtxInitID, log) 723 724 initMode, err := getInitMode(ctx, kbCtx, params, log) 725 if err != nil { 726 return nil, err 727 } 728 729 config := NewConfigLocal(initMode, 730 func(module string) logger.Logger { 731 mname := logPrefix 732 if module != "" { 733 mname += fmt.Sprintf("(%s)", module) 734 } 735 lg := logger.New(mname) 736 if params.Debug { 737 // Turn on debugging. TODO: allow a proper log file and 738 // style to be specified. 739 lg.Configure("", true, "") 740 } 741 return lg 742 }, params.StorageRoot, params.DiskCacheMode, kbCtx) 743 config.SetVLogLevel(kbCtx.GetVDebugSetting()) 744 745 if cap := getCleanBlockCacheCapacity(ctx, kbCtx, params, log); cap > 0 { 746 log.CDebugf( 747 ctx, "Overriding default clean block cache capacity from %d to %d", 748 config.BlockCache().GetCleanBytesCapacity(), cap) 749 config.BlockCache().SetCleanBytesCapacity(cap) 750 } 751 752 workers := config.Mode().BlockWorkers() 753 prefetchWorkers := config.Mode().PrefetchWorkers() 754 throttledPrefetchPeriod := config.Mode().ThrottledPrefetchPeriod() 755 config.SetBlockOps(NewBlockOpsStandard( 756 config, workers, prefetchWorkers, throttledPrefetchPeriod, kbCtx)) 757 758 bsplitter, err := data.NewBlockSplitterSimple( 759 data.MaxBlockSizeBytesDefault, 8*1024, config.Codec()) 760 if err != nil { 761 return nil, err 762 } 763 err = bsplitter.SetMaxDirEntriesByBlockSize(config.Codec()) 764 if err != nil { 765 return nil, err 766 } 767 config.SetBlockSplitter(bsplitter) 768 769 if registry := config.MetricsRegistry(); registry != nil { 770 keyCache := config.KeyCache() 771 keyCache = NewKeyCacheMeasured(keyCache, registry) 772 config.SetKeyCache(keyCache) 773 774 keyBundleCache := config.KeyBundleCache() 775 keyBundleCache = libkey.NewKeyBundleCacheMeasured( 776 keyBundleCache, registry) 777 config.SetKeyBundleCache(keyBundleCache) 778 } 779 780 config.SetMetadataVersion(params.MetadataVersion) 781 config.SetBlockCryptVersion(params.BlockCryptVersion) 782 config.SetTLFValidDuration(params.TLFValidDuration) 783 config.SetBGFlushPeriod(params.BGFlushPeriod) 784 785 kbfsLog := config.MakeLogger("") 786 787 // Initialize Keybase service connection. This needs to happen before 788 // KBPKI client. 789 if keybaseServiceCn == nil { 790 keybaseServiceCn = keybaseDaemon{} 791 } 792 service, err := keybaseServiceCn.NewKeybaseService( 793 config, params, kbCtx, kbfsLog) 794 if err != nil { 795 return nil, fmt.Errorf("problem creating service: %s", err) 796 } 797 if registry := config.MetricsRegistry(); registry != nil { 798 service = NewKeybaseServiceMeasured(service, registry) 799 } 800 config.SetKeybaseService(service) 801 802 // Initialize KBPKI client (needed for KBFSOps, MD Server, and Chat). 803 k := NewKBPKIClient(config, kbfsLog) 804 config.SetKBPKI(k) 805 806 // Initialize Chat client (for file edit notifications). 807 chat, err := keybaseServiceCn.NewChat(config, params, kbCtx, kbfsLog) 808 if err != nil { 809 return nil, fmt.Errorf("problem creating chat: %s", err) 810 } 811 config.SetChat(chat) 812 813 initDoneCh := make(chan struct{}) 814 kbfsOps := NewKBFSOpsStandard(kbCtx, config, initDoneCh) 815 defer close(initDoneCh) 816 config.SetKBFSOps(kbfsOps) 817 config.SetNotifier(kbfsOps) 818 config.SetKeyManager(NewKeyManagerStandard(config)) 819 config.SetMDOps(NewMDOpsStandard(config)) 820 821 config.SetDiskBlockCacheFraction(getCacheFrac( 822 ctx, kbCtx, params.DiskBlockCacheFraction, 823 defaultDiskBlockCacheFraction, configBlockCacheDiskMaxFracStr, log)) 824 config.SetSyncBlockCacheFraction(getCacheFrac( 825 ctx, kbCtx, params.SyncBlockCacheFraction, 826 defaultSyncBlockCacheFraction, configBlockCacheSyncMaxFracStr, log)) 827 err = config.EnableDiskLimiter(params.StorageRoot) 828 if err != nil { 829 log.CWarningf(ctx, "Could not enable disk limiter: %+v", err) 830 return nil, err 831 } 832 833 kbfsOps.favs.Initialize(ctx) 834 835 config.SetReporter(NewReporterKBPKI(config, 10, 1000)) 836 837 // Initialize Crypto client (needed for MD and Block servers). 838 crypto, err := keybaseServiceCn.NewCrypto(config, params, kbCtx, kbfsLog) 839 if err != nil { 840 return nil, fmt.Errorf("problem creating crypto: %s", err) 841 } 842 config.SetCrypto(crypto) 843 844 // Initialize MDServer connection. 845 mdServer, err := makeMDServer(kbCtx, config, params.MDServerAddr, kbCtx.NewRPCLogFactory(), log) 846 if err != nil { 847 return nil, fmt.Errorf("problem creating MD server: %+v", err) 848 } 849 config.SetMDServer(mdServer) 850 851 // Must do this after setting the md server, since it depends on 852 // being able to fetch MDs. 853 go kbfsOps.initSyncedTlfs() 854 855 // Initialize KeyServer connection. MDServer is the KeyServer at the 856 // moment. 857 keyServer, err := makeKeyServer(config, params.MDServerAddr, log) 858 if err != nil { 859 return nil, fmt.Errorf("problem creating key server: %+v", err) 860 } 861 if registry := config.MetricsRegistry(); registry != nil { 862 keyServer = libkey.NewKeyServerMeasured(keyServer, registry) 863 } 864 config.SetKeyServer(keyServer) 865 866 // Initialize BlockServer connection. 867 bserv, err := makeBlockServer( 868 kbCtx, config, params.BServerAddr, kbCtx.NewRPCLogFactory(), log) 869 if err != nil { 870 return nil, fmt.Errorf("cannot open block database: %+v", err) 871 } 872 if registry := config.MetricsRegistry(); registry != nil { 873 bserv = NewBlockServerMeasured(bserv, registry) 874 } 875 config.SetBlockServer(bserv) 876 877 // Don't trigger cache initialization warnings in single-op mode 878 // (e.g., during a git pull), because KBFS might be running in 879 // constrained mode and the cache remote proxy wouldn't be 880 // available (which is fine). 881 doSendWarnings := !config.Mode().IsSingleOp() 882 err = config.MakeDiskBlockCacheIfNotExists() 883 if err != nil { 884 log.CWarningf(ctx, "Could not initialize disk cache: %+v", err) 885 config.GetPerfLog().CDebugf( 886 ctx, "KBFS could not initialize disk cache: %v", err) 887 notification := &keybase1.FSNotification{ 888 StatusCode: keybase1.FSStatusCode_ERROR, 889 NotificationType: keybase1.FSNotificationType_INITIALIZED, 890 ErrorType: keybase1.FSErrorType_DISK_CACHE_ERROR_LOG_SEND, 891 } 892 if doSendWarnings { 893 defer config.Reporter().Notify(ctx, notification) 894 } 895 } else { 896 log.CDebugf(ctx, "Disk cache of type \"%s\" enabled", 897 params.DiskCacheMode.String()) 898 } 899 900 err = config.MakeDiskMDCacheIfNotExists() 901 if err != nil { 902 log.CWarningf(ctx, "Could not initialize MD cache: %+v", err) 903 notification := &keybase1.FSNotification{ 904 StatusCode: keybase1.FSStatusCode_ERROR, 905 NotificationType: keybase1.FSNotificationType_INITIALIZED, 906 ErrorType: keybase1.FSErrorType_DISK_CACHE_ERROR_LOG_SEND, 907 } 908 if doSendWarnings { 909 defer config.Reporter().Notify(ctx, notification) 910 } 911 } else { 912 log.CDebugf(ctx, "Disk MD cache enabled") 913 } 914 915 err = config.MakeDiskQuotaCacheIfNotExists() 916 if err != nil { 917 log.CWarningf(ctx, "Could not initialize disk quota cache: %+v", err) 918 notification := &keybase1.FSNotification{ 919 StatusCode: keybase1.FSStatusCode_ERROR, 920 NotificationType: keybase1.FSNotificationType_INITIALIZED, 921 ErrorType: keybase1.FSErrorType_DISK_CACHE_ERROR_LOG_SEND, 922 } 923 if doSendWarnings { 924 defer config.Reporter().Notify(ctx, notification) 925 } 926 } else { 927 log.CDebugf(ctx, "Disk quota cache enabled") 928 } 929 930 err = config.MakeBlockMetadataStoreIfNotExists() 931 if err != nil { 932 log.CWarningf(ctx, 933 "Could not initialize block metadata store: %+v", err) 934 /* 935 notification := &keybase1.FSNotification{ 936 StatusCode: keybase1.FSStatusCode_ERROR, 937 NotificationType: keybase1.FSNotificationType_INITIALIZED, 938 ErrorType: keybase1.FSErrorType_DISK_CACHE_ERROR_LOG_SEND, 939 } 940 defer config.Reporter().Notify(ctx, notification) 941 */ 942 } 943 log.CDebugf(ctx, "Disk block metadata store cache enabled") 944 945 if config.Mode().KBFSServiceEnabled() { 946 // Initialize kbfsService only when we run a full KBFS process. 947 // This requires the disk block cache to have been initialized, if it 948 // should be initialized. 949 kbfsService, err := NewKBFSService(kbCtx, config) 950 if err != nil { 951 // This error shouldn't be fatal 952 log.CWarningf(ctx, "Error starting RPC server for KBFS: %+v", err) 953 } else { 954 config.SetKBFSService(kbfsService) 955 log.CDebugf(ctx, "Started RPC server for KBFS") 956 } 957 } 958 959 ctx60s, cancel := context.WithTimeout(ctx, 60*time.Second) 960 defer cancel() 961 // TODO: Don't turn on journaling if either -bserver or 962 // -mdserver point to local implementations. 963 if params.EnableJournal && config.Mode().JournalEnabled() { 964 journalRoot := filepath.Join(params.StorageRoot, "kbfs_journal") 965 err = config.EnableJournaling(ctx60s, journalRoot, 966 params.TLFJournalBackgroundWorkStatus) 967 if err != nil { 968 log.CWarningf(ctx, "Could not initialize journal server: %+v", err) 969 } 970 log.CDebugf(ctx, "Journaling enabled") 971 } 972 973 if params.BGFlushDirOpBatchSize < 1 { 974 return nil, fmt.Errorf( 975 "Illegal sync batch size: %d", params.BGFlushDirOpBatchSize) 976 } 977 log.CDebugf(ctx, "Enabling a dir op batch size of %d", 978 params.BGFlushDirOpBatchSize) 979 config.SetBGFlushDirOpBatchSize(params.BGFlushDirOpBatchSize) 980 981 if config.Mode().OldStorageRootCleaningEnabled() { 982 go cleanOldTempStorageRoots(config) 983 } 984 985 return config, nil 986 } 987 988 // Shutdown does any necessary shutdown tasks for libkbfs. Shutdown 989 // should be called at the end of main. 990 func Shutdown() {}