github.com/keybase/client/go@v0.0.0-20240309051027-028f7c731f8b/kbfs/libkbfs/config_local.go (about) 1 // Copyright 2016 Keybase Inc. All rights reserved. 2 // Use of this source code is governed by a BSD 3 // license that can be found in the LICENSE file. 4 5 package libkbfs 6 7 import ( 8 "flag" 9 "os" 10 "path/filepath" 11 "strings" 12 "sync" 13 "time" 14 15 "github.com/keybase/client/go/kbfs/cache" 16 "github.com/keybase/client/go/kbfs/data" 17 "github.com/keybase/client/go/kbfs/idutil" 18 "github.com/keybase/client/go/kbfs/ioutil" 19 "github.com/keybase/client/go/kbfs/kbfscodec" 20 "github.com/keybase/client/go/kbfs/kbfscrypto" 21 "github.com/keybase/client/go/kbfs/kbfsedits" 22 "github.com/keybase/client/go/kbfs/kbfsmd" 23 "github.com/keybase/client/go/kbfs/ldbutils" 24 "github.com/keybase/client/go/kbfs/libkey" 25 "github.com/keybase/client/go/kbfs/tlf" 26 "github.com/keybase/client/go/libkb" 27 "github.com/keybase/client/go/logger" 28 "github.com/keybase/client/go/protocol/keybase1" 29 "github.com/pkg/errors" 30 metrics "github.com/rcrowley/go-metrics" 31 "github.com/shirou/gopsutil/mem" 32 "github.com/syndtr/goleveldb/leveldb" 33 "github.com/syndtr/goleveldb/leveldb/storage" 34 "golang.org/x/net/context" 35 "golang.org/x/net/trace" 36 ) 37 38 const ( 39 // Default time after setting the rekey bit before prompting for a 40 // paper key. 41 rekeyWithPromptWaitTimeDefault = 10 * time.Minute 42 // see Config doc for the purpose of DelayedCancellationGracePeriod 43 delayedCancellationGracePeriodDefault = 2 * time.Second 44 // tlfValidDurationDefault is the default for tlf validity before redoing identify. 45 tlfValidDurationDefault = 6 * time.Hour 46 // bgFlushDirOpThresholdDefault is the default for how many 47 // directory operations should be batched together in a single 48 // background flush. 49 bgFlushDirOpBatchSizeDefault = 100 50 // bgFlushPeriodDefault is the default for how long to wait for a 51 // batch to fill up before syncing a set of changes to the servers. 52 bgFlushPeriodDefault = 1 * time.Second 53 keyBundlesCacheCapacityBytes = 10 * cache.MB 54 // folder name for persisted config parameters. 55 syncedTlfConfigFolderName = "synced_tlf_config" 56 57 // By default, this will be the block type given to all blocks 58 // that aren't explicitly some other type. 59 defaultBlockTypeDefault = keybase1.BlockType_DATA 60 61 // By default, allow 10% of the free bytes on disk to be used in the disk block cache. 62 defaultDiskBlockCacheFraction = 0.10 63 64 // By default, allow 100% of the free bytes on disk to be used in the sync 65 // block cache. 66 defaultSyncBlockCacheFraction = 1.00 67 68 // By default, use v2 block encryption. 69 defaultBlockCryptVersion = kbfscrypto.EncryptionSecretboxWithKeyNonce 70 71 // How many times to retry loading the sync DB in the background 72 // if there's an error. 73 maxSyncDBLoadAttempts = 10 74 // How long to wait between load attempts. 75 syncDBLoadWaitPeriod = 10 * time.Second 76 ) 77 78 // ConfigLocal implements the Config interface using purely local 79 // server objects (no KBFS operations used RPCs). 80 type ConfigLocal struct { 81 lock sync.RWMutex 82 kbfs KBFSOps 83 keyman KeyManager 84 rep Reporter 85 kcache KeyCache 86 kbcache kbfsmd.KeyBundleCache 87 bcache data.BlockCache 88 dirtyBcache data.DirtyBlockCache 89 diskBlockCache DiskBlockCache 90 diskMDCache DiskMDCache 91 diskQuotaCache DiskQuotaCache 92 blockMetadataStore BlockMetadataStore 93 xattrStore XattrStore 94 codec kbfscodec.Codec 95 mdops MDOps 96 kops libkey.KeyOps 97 crypto Crypto 98 chat Chat 99 mdcache MDCache 100 bops BlockOps 101 mdserv MDServer 102 bserv BlockServer 103 keyserv libkey.KeyServer 104 service KeybaseService 105 bsplit data.BlockSplitter 106 notifier Notifier 107 clock Clock 108 kbpki KBPKI 109 renamer ConflictRenamer 110 userHistory *kbfsedits.UserHistory 111 registry metrics.Registry 112 loggerFn func(prefix string) logger.Logger 113 noBGFlush bool // logic opposite so the default value is the common setting 114 rwpWaitTime time.Duration 115 diskLimiter DiskLimiter 116 syncedTlfs map[tlf.ID]FolderSyncConfig // if nil, couldn't load DB 117 syncedTlfPaths map[string]bool 118 defaultBlockType keybase1.BlockType 119 kbfsService *KBFSService 120 kbCtx Context 121 rootNodeWrappers []func(Node) Node 122 tlfClearCancels map[tlf.ID]context.CancelFunc 123 vdebugSetting string 124 vlogs []*libkb.VDebugLog 125 126 maxNameBytes uint32 127 rekeyQueue RekeyQueue 128 storageRoot string 129 diskCacheMode DiskCacheMode 130 diskBlockCacheFraction float64 131 syncBlockCacheFraction float64 132 133 traceLock sync.RWMutex 134 traceEnabled bool 135 136 delayedCancellationGracePeriod time.Duration 137 138 // allKnownConfigsForTesting is used for testing, and contains all created 139 // Config objects in this test. 140 allKnownConfigsForTesting *[]Config 141 142 // tlfValidDuration is the time TLFs are valid before redoing identification. 143 tlfValidDuration time.Duration 144 145 // bgFlushDirOpBatchSize indicates how many directory operations 146 // should be batched together in a single background flush. 147 bgFlushDirOpBatchSize int 148 149 // bgFlushPeriod indicates how long to wait for a batch to fill up 150 // before syncing a set of changes to the servers. 151 bgFlushPeriod time.Duration 152 153 // metadataVersion is the version to use when creating new metadata. 154 metadataVersion kbfsmd.MetadataVer 155 156 // blockCryptVersion is the version to use when encrypting blocks. 157 blockCryptVersion kbfscrypto.EncryptionVer 158 159 // conflictResolutionDB stores information about failed CRs 160 conflictResolutionDB *ldbutils.LevelDb 161 162 // settingsDB stores information about local KBFS settings 163 settingsDB *SettingsDB 164 165 mode InitMode 166 167 quotaUsage map[keybase1.UserOrTeamID]*EventuallyConsistentQuotaUsage 168 rekeyFSMLimiter *OngoingWorkLimiter 169 170 subscriptionManagerManager *subscriptionManagerManager 171 } 172 173 // DiskCacheMode represents the mode of initialization for the disk cache. 174 type DiskCacheMode int 175 176 var _ flag.Value = (*DiskCacheMode)(nil) 177 178 const ( 179 // DiskCacheModeOff indicates to leave off the disk cache. 180 DiskCacheModeOff DiskCacheMode = iota 181 // DiskCacheModeLocal indicates to use a local disk cache. 182 DiskCacheModeLocal 183 // DiskCacheModeRemote indicates to use a remote disk cache. 184 DiskCacheModeRemote 185 ) 186 187 // String outputs a human-readable description of this DiskBlockCacheMode. 188 func (m DiskCacheMode) String() string { 189 switch m { 190 case DiskCacheModeOff: 191 return "off" 192 case DiskCacheModeLocal: 193 return "local" 194 case DiskCacheModeRemote: 195 return "remote" 196 } 197 return "unknown" 198 } 199 200 // Set parses a string representing a disk block cache initialization mode, 201 // and outputs the mode value corresponding to that string. Defaults to 202 // DiskCacheModeOff. 203 func (m *DiskCacheMode) Set(s string) error { 204 *m = DiskCacheModeOff 205 switch strings.ToLower(strings.TrimSpace(s)) { 206 case "local": 207 *m = DiskCacheModeLocal 208 case "remote": 209 *m = DiskCacheModeRemote 210 } 211 return nil 212 } 213 214 var _ Config = (*ConfigLocal)(nil) 215 216 // getDefaultCleanBlockCacheCapacity returns the default clean block 217 // cache capacity. If we can get total RAM of the system, we cap at 218 // the smaller of <1/8 of available memory> and 219 // <MaxBlockSizeBytesDefault * DefaultBlocksInMemCache>; otherwise, 220 // fallback to latter. 221 func getDefaultCleanBlockCacheCapacity(mode InitMode) uint64 { 222 const minCapacity = 10 * uint64(data.MaxBlockSizeBytesDefault) // 5mb 223 capacity := uint64(data.MaxBlockSizeBytesDefault) * DefaultBlocksInMemCache 224 vmstat, err := mem.VirtualMemory() 225 if err == nil { 226 ramBased := vmstat.Total / 8 227 if ramBased < minCapacity { 228 ramBased = minCapacity 229 } 230 if ramBased < capacity { 231 capacity = ramBased 232 } 233 } 234 if capacity > mode.MaxCleanBlockCacheCapacity() { 235 capacity = mode.MaxCleanBlockCacheCapacity() 236 } 237 return capacity 238 } 239 240 type keyOpsConfigWrapper struct { 241 Config 242 } 243 244 func (k keyOpsConfigWrapper) KBPKI() idutil.KBPKI { 245 return k.Config.KBPKI() 246 } 247 248 // NewConfigLocal constructs a new ConfigLocal with some default 249 // components that don't depend on a logger. The caller will have to 250 // fill in the rest. 251 // 252 // TODO: Now that NewConfigLocal takes loggerFn, add more default 253 // components. 254 func NewConfigLocal(mode InitMode, 255 loggerFn func(module string) logger.Logger, 256 storageRoot string, diskCacheMode DiskCacheMode, 257 kbCtx Context) *ConfigLocal { 258 config := &ConfigLocal{ 259 loggerFn: loggerFn, 260 storageRoot: storageRoot, 261 mode: mode, 262 diskCacheMode: diskCacheMode, 263 kbCtx: kbCtx, 264 tlfClearCancels: make(map[tlf.ID]context.CancelFunc), 265 } 266 config.SetCodec(kbfscodec.NewMsgpack()) 267 if diskCacheMode == DiskCacheModeLocal { 268 // Any error is logged in the function itself. 269 _ = config.loadSyncedTlfsLocked() 270 } 271 config.SetClock(data.WallClock{}) 272 config.SetReporter(NewReporterSimple(config.Clock(), 10)) 273 config.SetConflictRenamer(WriterDeviceDateConflictRenamer{config}) 274 config.ResetCaches() 275 config.SetKeyOps(libkey.NewKeyOpsStandard(keyOpsConfigWrapper{config})) 276 config.SetRekeyQueue(NewRekeyQueueStandard(config)) 277 uhLog := config.MakeLogger("HIS") 278 config.SetUserHistory(kbfsedits.NewUserHistory( 279 uhLog, config.MakeVLogger(uhLog))) 280 281 config.maxNameBytes = data.MaxNameBytesDefault 282 config.rwpWaitTime = rekeyWithPromptWaitTimeDefault 283 284 config.delayedCancellationGracePeriod = delayedCancellationGracePeriodDefault 285 // Don't bother creating the registry if UseNilMetrics is set, or 286 // if we're in minimal mode. 287 if !metrics.UseNilMetrics && config.Mode().MetricsEnabled() { 288 registry := metrics.NewRegistry() 289 config.SetMetricsRegistry(registry) 290 } 291 292 config.tlfValidDuration = tlfValidDurationDefault 293 config.bgFlushDirOpBatchSize = bgFlushDirOpBatchSizeDefault 294 config.bgFlushPeriod = bgFlushPeriodDefault 295 config.metadataVersion = defaultClientMetadataVer 296 config.defaultBlockType = defaultBlockTypeDefault 297 config.quotaUsage = 298 make(map[keybase1.UserOrTeamID]*EventuallyConsistentQuotaUsage) 299 config.rekeyFSMLimiter = NewOngoingWorkLimiter(config.Mode().RekeyWorkers()) 300 config.diskBlockCacheFraction = defaultDiskBlockCacheFraction 301 config.syncBlockCacheFraction = defaultSyncBlockCacheFraction 302 303 config.blockCryptVersion = defaultBlockCryptVersion 304 305 config.conflictResolutionDB = openCRDB(config) 306 config.settingsDB = openSettingsDB(config) 307 308 config.subscriptionManagerManager = newSubscriptionManagerManager(config) 309 310 return config 311 } 312 313 // KBFSOps implements the Config interface for ConfigLocal. 314 func (c *ConfigLocal) KBFSOps() KBFSOps { 315 c.lock.RLock() 316 defer c.lock.RUnlock() 317 return c.kbfs 318 } 319 320 // SetKBFSOps implements the Config interface for ConfigLocal. 321 func (c *ConfigLocal) SetKBFSOps(k KBFSOps) { 322 c.lock.Lock() 323 defer c.lock.Unlock() 324 c.kbfs = k 325 } 326 327 // KBPKI implements the Config interface for ConfigLocal. 328 func (c *ConfigLocal) KBPKI() KBPKI { 329 c.lock.RLock() 330 defer c.lock.RUnlock() 331 return c.kbpki 332 } 333 334 // CurrentSessionGetter implements the Config interface for ConfigLocal. 335 func (c *ConfigLocal) CurrentSessionGetter() idutil.CurrentSessionGetter { 336 c.lock.RLock() 337 defer c.lock.RUnlock() 338 return c.kbpki 339 } 340 341 // SetKBPKI implements the Config interface for ConfigLocal. 342 func (c *ConfigLocal) SetKBPKI(k KBPKI) { 343 c.lock.Lock() 344 defer c.lock.Unlock() 345 c.kbpki = k 346 } 347 348 // KeyManager implements the Config interface for ConfigLocal. 349 func (c *ConfigLocal) KeyManager() KeyManager { 350 c.lock.RLock() 351 defer c.lock.RUnlock() 352 return c.keyman 353 } 354 355 // SetKeyManager implements the Config interface for ConfigLocal. 356 func (c *ConfigLocal) SetKeyManager(k KeyManager) { 357 c.lock.Lock() 358 defer c.lock.Unlock() 359 c.keyman = k 360 } 361 362 // KeyGetter implements the Config interface for ConfigLocal. 363 func (c *ConfigLocal) keyGetter() blockKeyGetter { 364 c.lock.RLock() 365 defer c.lock.RUnlock() 366 return c.keyman 367 } 368 369 // Reporter implements the Config interface for ConfigLocal. 370 func (c *ConfigLocal) Reporter() Reporter { 371 c.lock.RLock() 372 defer c.lock.RUnlock() 373 return c.rep 374 } 375 376 // SetReporter implements the Config interface for ConfigLocal. 377 func (c *ConfigLocal) SetReporter(r Reporter) { 378 c.lock.Lock() 379 defer c.lock.Unlock() 380 c.rep = r 381 } 382 383 // KeyCache implements the Config interface for ConfigLocal. 384 func (c *ConfigLocal) KeyCache() KeyCache { 385 c.lock.RLock() 386 defer c.lock.RUnlock() 387 return c.kcache 388 } 389 390 // SetKeyCache implements the Config interface for ConfigLocal. 391 func (c *ConfigLocal) SetKeyCache(k KeyCache) { 392 c.lock.Lock() 393 defer c.lock.Unlock() 394 c.kcache = k 395 } 396 397 // KeyBundleCache implements the Config interface for ConfigLocal. 398 func (c *ConfigLocal) KeyBundleCache() kbfsmd.KeyBundleCache { 399 c.lock.RLock() 400 defer c.lock.RUnlock() 401 return c.kbcache 402 } 403 404 // SetKeyBundleCache implements the Config interface for ConfigLocal. 405 func (c *ConfigLocal) SetKeyBundleCache(k kbfsmd.KeyBundleCache) { 406 c.lock.Lock() 407 defer c.lock.Unlock() 408 c.kbcache = k 409 } 410 411 // BlockCache implements the Config interface for ConfigLocal. 412 func (c *ConfigLocal) BlockCache() data.BlockCache { 413 c.lock.RLock() 414 defer c.lock.RUnlock() 415 return c.bcache 416 } 417 418 // SetBlockCache implements the Config interface for ConfigLocal. 419 func (c *ConfigLocal) SetBlockCache(b data.BlockCache) { 420 c.lock.Lock() 421 defer c.lock.Unlock() 422 c.bcache = b 423 } 424 425 // DirtyBlockCache implements the Config interface for ConfigLocal. 426 func (c *ConfigLocal) DirtyBlockCache() data.DirtyBlockCache { 427 c.lock.RLock() 428 defer c.lock.RUnlock() 429 return c.dirtyBcache 430 } 431 432 // SetDirtyBlockCache implements the Config interface for ConfigLocal. 433 func (c *ConfigLocal) SetDirtyBlockCache(d data.DirtyBlockCache) { 434 c.lock.Lock() 435 defer c.lock.Unlock() 436 c.dirtyBcache = d 437 } 438 439 // DiskBlockCache implements the Config interface for ConfigLocal. 440 func (c *ConfigLocal) DiskBlockCache() DiskBlockCache { 441 c.lock.RLock() 442 defer c.lock.RUnlock() 443 return c.diskBlockCache 444 } 445 446 // SetDiskBlockCacheFraction implements the Config interface for ConfigLocal. 447 func (c *ConfigLocal) SetDiskBlockCacheFraction(fraction float64) { 448 c.lock.Lock() 449 defer c.lock.Unlock() 450 c.diskBlockCacheFraction = fraction 451 } 452 453 // SetSyncBlockCacheFraction implements the Config interface for ConfigLocal. 454 func (c *ConfigLocal) SetSyncBlockCacheFraction(fraction float64) { 455 c.lock.Lock() 456 defer c.lock.Unlock() 457 c.syncBlockCacheFraction = fraction 458 } 459 460 // DiskMDCache implements the Config interface for ConfigLocal. 461 func (c *ConfigLocal) DiskMDCache() DiskMDCache { 462 c.lock.RLock() 463 defer c.lock.RUnlock() 464 return c.diskMDCache 465 } 466 467 // DiskQuotaCache implements the Config interface for ConfigLocal. 468 func (c *ConfigLocal) DiskQuotaCache() DiskQuotaCache { 469 c.lock.RLock() 470 defer c.lock.RUnlock() 471 return c.diskQuotaCache 472 } 473 474 // XattrStore implements the Config interface for ConfigLocal. 475 func (c *ConfigLocal) XattrStore() XattrStore { 476 c.lock.RLock() 477 defer c.lock.RUnlock() 478 return c.xattrStore 479 } 480 481 // DiskLimiter implements the Config interface for ConfigLocal. 482 func (c *ConfigLocal) DiskLimiter() DiskLimiter { 483 c.lock.RLock() 484 defer c.lock.RUnlock() 485 return c.diskLimiter 486 } 487 488 // Crypto implements the Config interface for ConfigLocal. 489 func (c *ConfigLocal) Crypto() Crypto { 490 c.lock.RLock() 491 defer c.lock.RUnlock() 492 return c.crypto 493 } 494 495 // Chat implements the Config interface for ConfigLocal. 496 func (c *ConfigLocal) Chat() Chat { 497 c.lock.RLock() 498 defer c.lock.RUnlock() 499 return c.chat 500 } 501 502 // Signer implements the Config interface for ConfigLocal. 503 func (c *ConfigLocal) Signer() kbfscrypto.Signer { 504 c.lock.RLock() 505 defer c.lock.RUnlock() 506 return c.crypto 507 } 508 509 // SetCrypto implements the Config interface for ConfigLocal. 510 func (c *ConfigLocal) SetCrypto(cr Crypto) { 511 c.lock.Lock() 512 defer c.lock.Unlock() 513 c.crypto = cr 514 } 515 516 // SetChat implements the Config interface for ConfigLocal. 517 func (c *ConfigLocal) SetChat(ch Chat) { 518 c.lock.Lock() 519 defer c.lock.Unlock() 520 c.chat = ch 521 } 522 523 // CryptoPure implements the Config interface for ConfigLocal. 524 func (c *ConfigLocal) cryptoPure() cryptoPure { 525 c.lock.RLock() 526 defer c.lock.RUnlock() 527 return c.crypto 528 } 529 530 // Codec implements the Config interface for ConfigLocal. 531 func (c *ConfigLocal) Codec() kbfscodec.Codec { 532 c.lock.RLock() 533 defer c.lock.RUnlock() 534 return c.codec 535 } 536 537 // SetCodec implements the Config interface for ConfigLocal. 538 func (c *ConfigLocal) SetCodec(co kbfscodec.Codec) { 539 c.lock.Lock() 540 defer c.lock.Unlock() 541 c.codec = co 542 RegisterOps(c.codec) 543 } 544 545 // MDOps implements the Config interface for ConfigLocal. 546 func (c *ConfigLocal) MDOps() MDOps { 547 c.lock.RLock() 548 defer c.lock.RUnlock() 549 return c.mdops 550 } 551 552 // SetMDOps implements the Config interface for ConfigLocal. 553 func (c *ConfigLocal) SetMDOps(m MDOps) { 554 c.lock.Lock() 555 defer c.lock.Unlock() 556 c.mdops = m 557 } 558 559 // KeyOps implements the Config interface for ConfigLocal. 560 func (c *ConfigLocal) KeyOps() libkey.KeyOps { 561 c.lock.RLock() 562 defer c.lock.RUnlock() 563 return c.kops 564 } 565 566 // SetKeyOps implements the Config interface for ConfigLocal. 567 func (c *ConfigLocal) SetKeyOps(k libkey.KeyOps) { 568 c.lock.Lock() 569 defer c.lock.Unlock() 570 c.kops = k 571 } 572 573 // MDCache implements the Config interface for ConfigLocal. 574 func (c *ConfigLocal) MDCache() MDCache { 575 c.lock.RLock() 576 defer c.lock.RUnlock() 577 return c.mdcache 578 } 579 580 // SetMDCache implements the Config interface for ConfigLocal. 581 func (c *ConfigLocal) SetMDCache(m MDCache) { 582 c.lock.Lock() 583 defer c.lock.Unlock() 584 c.mdcache = m 585 } 586 587 // BlockOps implements the Config interface for ConfigLocal. 588 func (c *ConfigLocal) BlockOps() BlockOps { 589 c.lock.RLock() 590 defer c.lock.RUnlock() 591 return c.bops 592 } 593 594 // SetBlockOps implements the Config interface for ConfigLocal. 595 func (c *ConfigLocal) SetBlockOps(b BlockOps) { 596 c.lock.Lock() 597 defer c.lock.Unlock() 598 c.bops = b 599 } 600 601 // MDServer implements the Config interface for ConfigLocal. 602 func (c *ConfigLocal) MDServer() MDServer { 603 c.lock.RLock() 604 defer c.lock.RUnlock() 605 return c.mdserv 606 } 607 608 // SetMDServer implements the Config interface for ConfigLocal. 609 func (c *ConfigLocal) SetMDServer(m MDServer) { 610 c.lock.Lock() 611 defer c.lock.Unlock() 612 c.mdserv = m 613 } 614 615 // BlockServer implements the Config interface for ConfigLocal. 616 func (c *ConfigLocal) BlockServer() BlockServer { 617 c.lock.RLock() 618 defer c.lock.RUnlock() 619 return c.bserv 620 } 621 622 // SetBlockServer implements the Config interface for ConfigLocal. 623 func (c *ConfigLocal) SetBlockServer(b BlockServer) { 624 c.lock.Lock() 625 defer c.lock.Unlock() 626 c.bserv = b 627 } 628 629 // KeyServer implements the Config interface for ConfigLocal. 630 func (c *ConfigLocal) KeyServer() libkey.KeyServer { 631 c.lock.RLock() 632 defer c.lock.RUnlock() 633 return c.keyserv 634 } 635 636 // SetKeyServer implements the Config interface for ConfigLocal. 637 func (c *ConfigLocal) SetKeyServer(k libkey.KeyServer) { 638 c.lock.Lock() 639 defer c.lock.Unlock() 640 c.keyserv = k 641 } 642 643 // KeybaseService implements the Config interface for ConfigLocal. 644 func (c *ConfigLocal) KeybaseService() KeybaseService { 645 c.lock.RLock() 646 defer c.lock.RUnlock() 647 return c.service 648 } 649 650 // SetKeybaseService implements the Config interface for ConfigLocal. 651 func (c *ConfigLocal) SetKeybaseService(k KeybaseService) { 652 c.lock.Lock() 653 defer c.lock.Unlock() 654 c.service = k 655 } 656 657 // BlockSplitter implements the Config interface for ConfigLocal. 658 func (c *ConfigLocal) BlockSplitter() data.BlockSplitter { 659 c.lock.RLock() 660 defer c.lock.RUnlock() 661 return c.bsplit 662 } 663 664 // SetBlockSplitter implements the Config interface for ConfigLocal. 665 func (c *ConfigLocal) SetBlockSplitter(b data.BlockSplitter) { 666 c.lock.Lock() 667 defer c.lock.Unlock() 668 c.bsplit = b 669 } 670 671 // Notifier implements the Config interface for ConfigLocal. 672 func (c *ConfigLocal) Notifier() Notifier { 673 c.lock.RLock() 674 defer c.lock.RUnlock() 675 return c.notifier 676 } 677 678 // SetNotifier implements the Config interface for ConfigLocal. 679 func (c *ConfigLocal) SetNotifier(n Notifier) { 680 c.lock.Lock() 681 defer c.lock.Unlock() 682 c.notifier = n 683 } 684 685 // Clock implements the Config interface for ConfigLocal. 686 func (c *ConfigLocal) Clock() Clock { 687 c.lock.RLock() 688 defer c.lock.RUnlock() 689 return c.clock 690 } 691 692 // SetClock implements the Config interface for ConfigLocal. 693 func (c *ConfigLocal) SetClock(cl Clock) { 694 c.lock.Lock() 695 defer c.lock.Unlock() 696 c.clock = cl 697 } 698 699 // ConflictRenamer implements the Config interface for ConfigLocal. 700 func (c *ConfigLocal) ConflictRenamer() ConflictRenamer { 701 c.lock.RLock() 702 defer c.lock.RUnlock() 703 return c.renamer 704 } 705 706 // SetConflictRenamer implements the Config interface for ConfigLocal. 707 func (c *ConfigLocal) SetConflictRenamer(cr ConflictRenamer) { 708 c.lock.Lock() 709 defer c.lock.Unlock() 710 c.renamer = cr 711 } 712 713 // UserHistory implements the Config interface for ConfigLocal. 714 func (c *ConfigLocal) UserHistory() *kbfsedits.UserHistory { 715 c.lock.RLock() 716 defer c.lock.RUnlock() 717 return c.userHistory 718 } 719 720 // SetUserHistory implements the Config interface for ConfigLocal. 721 func (c *ConfigLocal) SetUserHistory(uh *kbfsedits.UserHistory) { 722 c.lock.Lock() 723 defer c.lock.Unlock() 724 c.userHistory = uh 725 } 726 727 // MetadataVersion implements the Config interface for ConfigLocal. 728 func (c *ConfigLocal) MetadataVersion() kbfsmd.MetadataVer { 729 c.lock.RLock() 730 defer c.lock.RUnlock() 731 return c.metadataVersion 732 } 733 734 // SetMetadataVersion implements the Config interface for ConfigLocal. 735 func (c *ConfigLocal) SetMetadataVersion(mdVer kbfsmd.MetadataVer) { 736 c.lock.Lock() 737 defer c.lock.Unlock() 738 c.metadataVersion = mdVer 739 } 740 741 // DataVersion implements the Config interface for ConfigLocal. 742 func (c *ConfigLocal) DataVersion() data.Ver { 743 return data.IndirectDirsVer 744 } 745 746 // BlockCryptVersion implements the Config interface for ConfigLocal. 747 func (c *ConfigLocal) BlockCryptVersion() kbfscrypto.EncryptionVer { 748 c.lock.RLock() 749 defer c.lock.RUnlock() 750 return c.blockCryptVersion 751 } 752 753 // SetBlockCryptVersion implements the Config interface for ConfigLocal. 754 func (c *ConfigLocal) SetBlockCryptVersion(ver kbfscrypto.EncryptionVer) { 755 c.lock.Lock() 756 defer c.lock.Unlock() 757 c.blockCryptVersion = ver 758 } 759 760 // DefaultBlockType implements the Config interface for ConfigLocal. 761 func (c *ConfigLocal) DefaultBlockType() keybase1.BlockType { 762 c.lock.RLock() 763 defer c.lock.RUnlock() 764 return c.defaultBlockType 765 } 766 767 // SetDefaultBlockType implements the Config interface for ConfigLocal. 768 func (c *ConfigLocal) SetDefaultBlockType(blockType keybase1.BlockType) { 769 c.lock.Lock() 770 defer c.lock.Unlock() 771 c.defaultBlockType = blockType 772 } 773 774 // DoBackgroundFlushes implements the Config interface for ConfigLocal. 775 func (c *ConfigLocal) DoBackgroundFlushes() bool { 776 if !c.Mode().BackgroundFlushesEnabled() { 777 return false 778 } 779 780 c.lock.RLock() 781 defer c.lock.RUnlock() 782 return !c.noBGFlush 783 } 784 785 // SetDoBackgroundFlushes implements the Config interface for ConfigLocal. 786 func (c *ConfigLocal) SetDoBackgroundFlushes(doBGFlush bool) { 787 c.lock.Lock() 788 defer c.lock.Unlock() 789 c.noBGFlush = !doBGFlush 790 } 791 792 // RekeyWithPromptWaitTime implements the Config interface for 793 // ConfigLocal. 794 func (c *ConfigLocal) RekeyWithPromptWaitTime() time.Duration { 795 c.lock.Lock() 796 defer c.lock.Unlock() 797 return c.rwpWaitTime 798 } 799 800 // SetRekeyWithPromptWaitTime implements the Config interface for 801 // ConfigLocal. 802 func (c *ConfigLocal) SetRekeyWithPromptWaitTime(d time.Duration) { 803 c.lock.RLock() 804 defer c.lock.RUnlock() 805 c.rwpWaitTime = d 806 } 807 808 // Mode implements the Config interface for ConfigLocal. 809 func (c *ConfigLocal) Mode() InitMode { 810 c.lock.RLock() 811 defer c.lock.RUnlock() 812 return c.mode 813 } 814 815 // SetMode implements the Config interface for ConfigLocal. 816 func (c *ConfigLocal) SetMode(mode InitMode) { 817 c.lock.Lock() 818 defer c.lock.Unlock() 819 c.mode = mode 820 } 821 822 // IsTestMode implements the Config interface for ConfigLocal. 823 func (c *ConfigLocal) IsTestMode() bool { 824 c.lock.RLock() 825 defer c.lock.RUnlock() 826 return c.mode.IsTestMode() 827 } 828 829 // DelayedCancellationGracePeriod implements the Config interface for ConfigLocal. 830 func (c *ConfigLocal) DelayedCancellationGracePeriod() time.Duration { 831 return c.delayedCancellationGracePeriod 832 } 833 834 // SetDelayedCancellationGracePeriod implements the Config interface for ConfigLocal. 835 func (c *ConfigLocal) SetDelayedCancellationGracePeriod(d time.Duration) { 836 c.delayedCancellationGracePeriod = d 837 } 838 839 // ReqsBufSize implements the Config interface for ConfigLocal. 840 func (c *ConfigLocal) ReqsBufSize() int { 841 return 20 842 } 843 844 // MaxNameBytes implements the Config interface for ConfigLocal. 845 func (c *ConfigLocal) MaxNameBytes() uint32 { 846 return c.maxNameBytes 847 } 848 849 // SetStorageRoot sets the storage root directory for this config. 850 func (c *ConfigLocal) SetStorageRoot(storageRoot string) { 851 c.lock.Lock() 852 defer c.lock.Unlock() 853 c.storageRoot = storageRoot 854 } 855 856 // StorageRoot implements the Config interface for ConfigLocal. 857 func (c *ConfigLocal) StorageRoot() string { 858 c.lock.RLock() 859 defer c.lock.RUnlock() 860 return c.storageRoot 861 } 862 863 func (c *ConfigLocal) resetCachesWithoutShutdown() data.DirtyBlockCache { 864 c.lock.Lock() 865 defer c.lock.Unlock() 866 c.mdcache = NewMDCacheStandard(defaultMDCacheCapacity) 867 c.kcache = NewKeyCacheStandard(defaultMDCacheCapacity) 868 c.kbcache = kbfsmd.NewKeyBundleCacheLRU(keyBundlesCacheCapacityBytes) 869 870 log := c.MakeLogger("") 871 var capacity uint64 872 if c.bcache == nil { 873 capacity = getDefaultCleanBlockCacheCapacity(c.mode) 874 log.Debug("setting default clean block cache capacity to %d", 875 capacity) 876 } else { 877 capacity = c.bcache.GetCleanBytesCapacity() 878 log.Debug("setting clean block cache capacity based on existing value %d", 879 capacity) 880 } 881 c.bcache = data.NewBlockCacheStandard(10000, capacity) 882 883 if !c.mode.DirtyBlockCacheEnabled() { 884 return nil 885 } 886 887 oldDirtyBcache := c.dirtyBcache 888 889 // TODO: we should probably fail or re-schedule this reset if 890 // there is anything dirty in the dirty block cache. 891 892 // The minimum number of bytes we'll try to sync in parallel. 893 // This should be roughly the minimum amount of bytes we expect 894 // our worst supported connection to send within the timeout 895 // forced on us by the upper layer (19 seconds on OS X). With the 896 // current default of a single block, this minimum works out to 897 // ~1MB, so we can support a connection speed as low as ~54 KB/s. 898 minSyncBufferSize := int64(data.MaxBlockSizeBytesDefault) 899 900 // The maximum number of bytes we can try to sync at once (also limits the 901 // amount of memory used by dirty blocks). We use the same value from clean 902 // block cache capacity here. 903 maxSyncBufferSize := int64(capacity) 904 905 // Start off conservatively to avoid getting immediate timeouts on 906 // slow connections. 907 startSyncBufferSize := minSyncBufferSize 908 909 dbcLog := c.MakeLogger("DBC") 910 c.dirtyBcache = data.NewDirtyBlockCacheStandard( 911 c.clock, dbcLog, c.makeVLoggerLocked(dbcLog), 912 minSyncBufferSize, maxSyncBufferSize, startSyncBufferSize) 913 return oldDirtyBcache 914 } 915 916 // ResetCaches implements the Config interface for ConfigLocal. 917 func (c *ConfigLocal) ResetCaches() { 918 oldDirtyBcache := c.resetCachesWithoutShutdown() 919 jManager, err := GetJournalManager(c) 920 if err == nil { 921 if err := c.journalizeBcaches(jManager); err != nil { 922 if log := c.MakeLogger(""); log != nil { 923 log.CWarningf( 924 context.TODO(), 925 "Error journalizing dirty block cache: %+v", err) 926 } 927 } 928 } 929 if oldDirtyBcache != nil { 930 // Shutdown outside of the lock so it doesn't block other 931 // access to this config. 932 if err := oldDirtyBcache.Shutdown(); err != nil { 933 if log := c.MakeLogger(""); log != nil { 934 log.CWarningf( 935 context.TODO(), 936 "Error shutting down old dirty block cache: %+v", err) 937 } 938 } 939 } 940 } 941 942 // MakeLogger implements the logMaker interface for ConfigLocal. 943 func (c *ConfigLocal) MakeLogger(module string) logger.Logger { 944 // No need to lock since c.loggerFn is initialized once at 945 // construction. Also resetCachesWithoutShutdown would deadlock. 946 return c.loggerFn(module) 947 } 948 949 func (c *ConfigLocal) makeVLoggerLocked(log logger.Logger) *libkb.VDebugLog { 950 vlog := libkb.NewVDebugLog(log) 951 vlog.Configure(c.vdebugSetting) 952 c.vlogs = append(c.vlogs, vlog) 953 return vlog 954 } 955 956 // MakeVLogger implements the logMaker interface for ConfigLocal. 957 func (c *ConfigLocal) MakeVLogger(log logger.Logger) *libkb.VDebugLog { 958 c.lock.Lock() 959 defer c.lock.Unlock() 960 return c.makeVLoggerLocked(log) 961 } 962 963 // MetricsRegistry implements the Config interface for ConfigLocal. 964 func (c *ConfigLocal) MetricsRegistry() metrics.Registry { 965 return c.registry 966 } 967 968 // SetRekeyQueue implements the Config interface for ConfigLocal. 969 func (c *ConfigLocal) SetRekeyQueue(r RekeyQueue) { 970 c.rekeyQueue = r 971 } 972 973 // RekeyQueue implements the Config interface for ConfigLocal. 974 func (c *ConfigLocal) RekeyQueue() RekeyQueue { 975 return c.rekeyQueue 976 } 977 978 // SetMetricsRegistry implements the Config interface for ConfigLocal. 979 func (c *ConfigLocal) SetMetricsRegistry(r metrics.Registry) { 980 c.registry = r 981 } 982 983 // SetTraceOptions implements the Config interface for ConfigLocal. 984 func (c *ConfigLocal) SetTraceOptions(enabled bool) { 985 c.traceLock.Lock() 986 defer c.traceLock.Unlock() 987 c.traceEnabled = enabled 988 } 989 990 // MaybeStartTrace implements the Config interface for ConfigLocal. 991 func (c *ConfigLocal) MaybeStartTrace( 992 ctx context.Context, family, title string) context.Context { 993 traceEnabled := func() bool { 994 c.traceLock.RLock() 995 defer c.traceLock.RUnlock() 996 return c.traceEnabled 997 }() 998 if !traceEnabled { 999 return ctx 1000 } 1001 1002 tr := trace.New(family, title) 1003 tr.SetMaxEvents(25) 1004 ctx = trace.NewContext(ctx, tr) 1005 return ctx 1006 } 1007 1008 // MaybeFinishTrace implements the Config interface for ConfigLocal. 1009 func (c *ConfigLocal) MaybeFinishTrace(ctx context.Context, err error) { 1010 if tr, ok := trace.FromContext(ctx); ok { 1011 if err != nil { 1012 tr.LazyPrintf("err=%+v", err) 1013 tr.SetError() 1014 } 1015 tr.Finish() 1016 } 1017 } 1018 1019 // SetTLFValidDuration implements the Config interface for ConfigLocal. 1020 func (c *ConfigLocal) SetTLFValidDuration(r time.Duration) { 1021 c.tlfValidDuration = r 1022 } 1023 1024 // TLFValidDuration implements the Config interface for ConfigLocal. 1025 func (c *ConfigLocal) TLFValidDuration() time.Duration { 1026 return c.tlfValidDuration 1027 } 1028 1029 // SetBGFlushDirOpBatchSize implements the Config interface for ConfigLocal. 1030 func (c *ConfigLocal) SetBGFlushDirOpBatchSize(s int) { 1031 c.lock.Lock() 1032 defer c.lock.Unlock() 1033 c.bgFlushDirOpBatchSize = s 1034 } 1035 1036 // BGFlushDirOpBatchSize implements the Config interface for ConfigLocal. 1037 func (c *ConfigLocal) BGFlushDirOpBatchSize() int { 1038 c.lock.RLock() 1039 defer c.lock.RUnlock() 1040 return c.bgFlushDirOpBatchSize 1041 } 1042 1043 // SetBGFlushPeriod implements the Config interface for ConfigLocal. 1044 func (c *ConfigLocal) SetBGFlushPeriod(p time.Duration) { 1045 c.lock.Lock() 1046 defer c.lock.Unlock() 1047 c.bgFlushPeriod = p 1048 } 1049 1050 // BGFlushPeriod implements the Config interface for ConfigLocal. 1051 func (c *ConfigLocal) BGFlushPeriod() time.Duration { 1052 c.lock.RLock() 1053 defer c.lock.RUnlock() 1054 return c.bgFlushPeriod 1055 } 1056 1057 // Shutdown implements the Config interface for ConfigLocal. 1058 func (c *ConfigLocal) Shutdown(ctx context.Context) error { 1059 c.RekeyQueue().Shutdown() 1060 if c.CheckStateOnShutdown() && c.allKnownConfigsForTesting != nil { 1061 // Before we do anything, wait for all archiving and 1062 // journaling to finish. 1063 for _, config := range *c.allKnownConfigsForTesting { 1064 kbfsOps, ok := config.KBFSOps().(*KBFSOpsStandard) 1065 if !ok { 1066 continue 1067 } 1068 if err := kbfsOps.shutdownEdits(ctx); err != nil { 1069 return err 1070 } 1071 for _, fbo := range kbfsOps.ops { 1072 if err := fbo.fbm.waitForArchives(ctx); err != nil { 1073 return err 1074 } 1075 if err := fbo.fbm.waitForDeletingBlocks(ctx); err != nil { 1076 return err 1077 } 1078 log := config.MakeLogger("") 1079 if err := WaitForTLFJournal(ctx, config, fbo.id(), 1080 log); err != nil { 1081 return err 1082 } 1083 // The above wait could have resulted in some MD 1084 // flushes, so now we have to wait on any archives as 1085 // well. We only need one more check for this, since 1086 // archives don't produce MDs. 1087 if err := fbo.mdFlushes.Wait(ctx); err != nil { 1088 return err 1089 } 1090 if err := fbo.fbm.waitForArchives(ctx); err != nil { 1091 return err 1092 } 1093 if err := WaitForTLFJournal(ctx, config, fbo.id(), 1094 log); err != nil { 1095 return err 1096 } 1097 } 1098 } 1099 } 1100 1101 var errorList []error 1102 err := c.KBFSOps().Shutdown(ctx) 1103 if err != nil { 1104 errorList = append(errorList, err) 1105 // Continue with shutdown regardless of err. 1106 } 1107 err = c.BlockOps().Shutdown(ctx) 1108 if err != nil { 1109 errorList = append(errorList, err) 1110 } 1111 c.MDServer().Shutdown() 1112 c.KeyServer().Shutdown() 1113 c.KeybaseService().Shutdown() 1114 c.BlockServer().Shutdown(ctx) 1115 c.Crypto().Shutdown() 1116 c.Reporter().Shutdown() 1117 dirtyBcache := c.DirtyBlockCache() 1118 if dirtyBcache != nil { 1119 err = dirtyBcache.Shutdown() 1120 } 1121 if err != nil { 1122 errorList = append(errorList, err) 1123 } 1124 dbc := c.DiskBlockCache() 1125 if dbc != nil { 1126 dbc.Shutdown(ctx) 1127 } 1128 dmc := c.DiskMDCache() 1129 if dmc != nil { 1130 dmc.Shutdown(ctx) 1131 } 1132 dqc := c.DiskQuotaCache() 1133 if dqc != nil { 1134 dqc.Shutdown(ctx) 1135 } 1136 bms := c.blockMetadataStore 1137 if bms != nil { 1138 bms.Shutdown() 1139 } 1140 if c.conflictResolutionDB != nil { 1141 if err := c.conflictResolutionDB.Close(); err != nil { 1142 errorList = append(errorList, err) 1143 } 1144 } 1145 if c.settingsDB != nil { 1146 if err := c.settingsDB.Close(); err != nil { 1147 errorList = append(errorList, err) 1148 } 1149 } 1150 kbfsServ := c.kbfsService 1151 if kbfsServ != nil { 1152 kbfsServ.Shutdown() 1153 } 1154 1155 c.subscriptionManagerManager.Shutdown(ctx) 1156 1157 if len(errorList) == 1 { 1158 return errorList[0] 1159 } else if len(errorList) > 1 { 1160 // Aggregate errors 1161 return errors.Errorf("Multiple errors on shutdown: %+v", errorList) 1162 } 1163 1164 c.lock.Lock() 1165 defer c.lock.Unlock() 1166 for _, cancel := range c.tlfClearCancels { 1167 cancel() 1168 } 1169 1170 return nil 1171 } 1172 1173 // CheckStateOnShutdown implements the Config interface for ConfigLocal. 1174 func (c *ConfigLocal) CheckStateOnShutdown() bool { 1175 if !c.IsTestMode() { 1176 return false 1177 } 1178 if md, ok := c.MDServer().(mdServerLocal); ok { 1179 return !md.isShutdown() 1180 } 1181 return false 1182 } 1183 1184 func (c *ConfigLocal) journalizeBcaches(jManager *JournalManager) error { 1185 syncCache, ok := c.DirtyBlockCache().(*data.DirtyBlockCacheStandard) 1186 if !ok { 1187 return errors.Errorf("Dirty bcache unexpectedly type %T", syncCache) 1188 } 1189 jManager.delegateDirtyBlockCache = syncCache 1190 1191 // Make a dirty block cache specifically for the journal 1192 // server. Since this doesn't rely directly on the network, 1193 // there's no need for an adaptive sync buffer size, so we 1194 // always set the min and max to the same thing. 1195 maxSyncBufferSize := int64(ForcedBranchSquashBytesThresholdDefault) 1196 log := c.MakeLogger("DBCJ") 1197 journalCache := data.NewDirtyBlockCacheStandard( 1198 c.clock, log, c.MakeVLogger(log), 1199 maxSyncBufferSize, maxSyncBufferSize, maxSyncBufferSize) 1200 c.SetDirtyBlockCache(jManager.dirtyBlockCache(journalCache)) 1201 1202 jManager.delegateBlockCache = c.BlockCache() 1203 c.SetBlockCache(jManager.blockCache()) 1204 return nil 1205 } 1206 1207 // GetQuotaUsage implements the Config interface for ConfigLocal. 1208 func (c *ConfigLocal) GetQuotaUsage( 1209 chargedTo keybase1.UserOrTeamID) *EventuallyConsistentQuotaUsage { 1210 c.lock.RLock() 1211 quota, ok := c.quotaUsage[chargedTo] 1212 if ok { 1213 c.lock.RUnlock() 1214 return quota 1215 } 1216 c.lock.RUnlock() 1217 1218 c.lock.Lock() 1219 defer c.lock.Unlock() 1220 quota, ok = c.quotaUsage[chargedTo] 1221 log := c.MakeLogger(QuotaUsageLogModule("BDL")) 1222 vlog := c.makeVLoggerLocked(log) 1223 if !ok { 1224 if chargedTo.IsTeamOrSubteam() { 1225 quota = NewEventuallyConsistentTeamQuotaUsage( 1226 c, chargedTo.AsTeamOrBust(), log, vlog) 1227 } else { 1228 quota = NewEventuallyConsistentQuotaUsage(c, log, vlog) 1229 } 1230 c.quotaUsage[chargedTo] = quota 1231 } 1232 return quota 1233 } 1234 1235 // GetPerfLog returns the performance logger for KBFS. 1236 func (c *ConfigLocal) GetPerfLog() logger.Logger { 1237 perfLog := c.kbCtx.GetPerfLog() 1238 if perfLog != nil { 1239 return perfLog 1240 } 1241 return c.MakeLogger("") 1242 } 1243 1244 // EnableDiskLimiter fills in c.diskLimiter for use in journaling and 1245 // disk caching. It returns the EventuallyConsistentQuotaUsage object 1246 // used by the disk limiter. 1247 func (c *ConfigLocal) EnableDiskLimiter(configRoot string) error { 1248 if c.diskLimiter != nil { 1249 return errors.New("c.diskLimiter is already non-nil") 1250 } 1251 1252 params := makeDefaultBackpressureDiskLimiterParams( 1253 configRoot, c.GetQuotaUsage, c.diskBlockCacheFraction, c.syncBlockCacheFraction) 1254 log := c.MakeLogger("") 1255 log.Debug("Setting disk storage byte limit to %d and file limit to %d", 1256 params.byteLimit, params.fileLimit) 1257 err := os.MkdirAll(configRoot, 0700) 1258 if err != nil { 1259 return err 1260 } 1261 1262 diskLimiter, err := newBackpressureDiskLimiter(log, params) 1263 if err != nil { 1264 return err 1265 } 1266 c.diskLimiter = diskLimiter 1267 return nil 1268 } 1269 1270 // EnableJournaling creates a JournalManager and attaches it to 1271 // this config. journalRoot must be non-empty. Errors returned are 1272 // non-fatal. 1273 func (c *ConfigLocal) EnableJournaling( 1274 ctx context.Context, journalRoot string, 1275 bws TLFJournalBackgroundWorkStatus) error { 1276 jManager, err := GetJournalManager(c) 1277 if err == nil { 1278 // Journaling shouldn't be enabled twice for the same 1279 // config. 1280 return errors.New("trying to enable journaling twice") 1281 } 1282 1283 if c.diskLimiter == nil { 1284 return errors.New("disk limiter must be enabled to enable journaling") 1285 } 1286 1287 // TODO: Sanity-check the root directory, e.g. create 1288 // it if it doesn't exist, make sure that it doesn't 1289 // point to /keybase itself, etc. 1290 log := c.MakeLogger("") 1291 branchListener := c.KBFSOps().(branchChangeListener) 1292 flushListener := c.KBFSOps().(mdFlushListener) 1293 1294 // Make sure the journal root exists. 1295 err = ioutil.MkdirAll(journalRoot, 0700) 1296 if err != nil { 1297 return err 1298 } 1299 1300 jManager = makeJournalManager(c, log, journalRoot, c.BlockCache(), 1301 c.DirtyBlockCache(), c.BlockServer(), c.MDOps(), branchListener, 1302 flushListener, bws) 1303 1304 c.SetBlockServer(jManager.blockServer()) 1305 c.SetMDOps(jManager.mdOps()) 1306 1307 bcacheErr := c.journalizeBcaches(jManager) 1308 enableErr := func() error { 1309 // If this fails, then existing journals will be 1310 // enabled when we receive the login notification. 1311 session, err := c.KBPKI().GetCurrentSession(ctx) 1312 if err != nil { 1313 return err 1314 } 1315 1316 err = jManager.EnableExistingJournals( 1317 ctx, session.UID, session.VerifyingKey, bws) 1318 if err != nil { 1319 return err 1320 } 1321 1322 wg := jManager.MakeFBOsForExistingJournals(ctx) 1323 wg.Wait() 1324 return nil 1325 }() 1326 switch { 1327 case bcacheErr != nil && enableErr != nil: 1328 return errors.Errorf( 1329 "Got errors %+v and %+v", bcacheErr, enableErr) 1330 case bcacheErr != nil: 1331 return bcacheErr 1332 case enableErr != nil: 1333 return enableErr 1334 } 1335 1336 return nil 1337 } 1338 1339 func (c *ConfigLocal) cleanSyncBlockCacheForTlfInBackgroundLocked( 1340 tlfID tlf.ID, ch chan<- error) { 1341 // Start a background goroutine deleting all the blocks from this 1342 // TLF. 1343 ctx, cancel := context.WithCancel(context.Background()) 1344 if oldCancel, ok := c.tlfClearCancels[tlfID]; ok { 1345 oldCancel() 1346 } 1347 c.tlfClearCancels[tlfID] = cancel 1348 diskBlockCache := c.diskBlockCache 1349 go func() { 1350 defer cancel() 1351 ch <- diskBlockCache.ClearAllTlfBlocks( 1352 ctx, tlfID, DiskBlockSyncCache) 1353 }() 1354 } 1355 1356 func (c *ConfigLocal) cleanSyncBlockCache() { 1357 ctx := context.Background() 1358 dbc := c.DiskBlockCache() 1359 log := c.MakeLogger("") 1360 if dbc == nil { 1361 // This could happen if there's a race with the caches being 1362 // reset multiple times in a row. Hopefully the next reset 1363 // will clean the caches properly. 1364 log.CDebugf(ctx, "No cache set; skipping sync cleaning") 1365 return 1366 } 1367 err := dbc.WaitUntilStarted(DiskBlockSyncCache) 1368 if err != nil { 1369 log.CDebugf( 1370 ctx, "Disk block cache failed to start; can't clean: %+v", err) 1371 return 1372 } 1373 1374 c.lock.Lock() 1375 defer c.lock.Unlock() 1376 1377 for i := 0; c.syncedTlfs == nil && i < maxSyncDBLoadAttempts; i++ { 1378 // Sometimes transient iOS storage permission errors prevent 1379 // us from reading the sync config DB on startup. In that 1380 // case, the `syncedTlfs` map will be nil, but we don't want 1381 // to delete all the synced blocks. 1382 log.CDebugf( 1383 ctx, "Re-loading synced TLF list for cleaning") 1384 1385 err := c.loadSyncedTlfsLocked() 1386 if err == nil { 1387 break 1388 } 1389 c.lock.Unlock() 1390 time.Sleep(syncDBLoadWaitPeriod) 1391 c.lock.Lock() 1392 } 1393 if c.syncedTlfs == nil { 1394 log.CDebugf( 1395 ctx, "Couldn't load synced TLF list for cleaning; giving up") 1396 return 1397 } 1398 1399 cacheTlfIDs, err := c.diskBlockCache.GetTlfIDs(ctx, DiskBlockSyncCache) 1400 if err != nil { 1401 log.CDebugf( 1402 ctx, "Disk block cache can't get TLF IDs; can't clean: %+v", err) 1403 return 1404 } 1405 1406 for _, id := range cacheTlfIDs { 1407 if _, ok := c.syncedTlfs[id]; ok { 1408 continue 1409 } 1410 1411 c.GetPerfLog().CDebugf(ctx, "Clearing KBFS sync blocks for TLF %s", id) 1412 c.cleanSyncBlockCacheForTlfInBackgroundLocked(id, make(chan error, 1)) 1413 } 1414 } 1415 1416 func (c *ConfigLocal) resetDiskBlockCacheLocked() error { 1417 dbc, err := newDiskBlockCacheWrapped(c, c.storageRoot, c.mode) 1418 if err != nil { 1419 return err 1420 } 1421 c.diskBlockCache = dbc 1422 if !c.mode.IsTestMode() { 1423 go c.cleanSyncBlockCache() 1424 } 1425 return nil 1426 } 1427 1428 // MakeDiskBlockCacheIfNotExists implements the Config interface for 1429 // ConfigLocal. 1430 func (c *ConfigLocal) MakeDiskBlockCacheIfNotExists() error { 1431 c.lock.Lock() 1432 defer c.lock.Unlock() 1433 if c.diskBlockCache != nil { 1434 return nil 1435 } 1436 switch c.diskCacheMode { 1437 case DiskCacheModeOff: 1438 return nil 1439 case DiskCacheModeLocal: 1440 return c.resetDiskBlockCacheLocked() 1441 case DiskCacheModeRemote: 1442 dbc, err := NewDiskBlockCacheRemote(c.kbCtx, c) 1443 if err != nil { 1444 return err 1445 } 1446 c.diskBlockCache = dbc 1447 return nil 1448 } 1449 return nil 1450 } 1451 1452 func (c *ConfigLocal) resetDiskMDCacheLocked() error { 1453 dmc, err := newDiskMDCacheLocal(c, c.storageRoot, c.mode) 1454 if err != nil { 1455 return err 1456 } 1457 if c.diskMDCache != nil { 1458 c.diskMDCache.Shutdown(context.TODO()) 1459 } 1460 c.diskMDCache = dmc 1461 return nil 1462 } 1463 1464 // MakeDiskMDCacheIfNotExists implements the Config interface for 1465 // ConfigLocal. 1466 func (c *ConfigLocal) MakeDiskMDCacheIfNotExists() error { 1467 c.lock.Lock() 1468 defer c.lock.Unlock() 1469 if c.diskMDCache != nil { 1470 return nil 1471 } 1472 return c.resetDiskMDCacheLocked() 1473 } 1474 1475 func (c *ConfigLocal) resetDiskQuotaCacheLocked() error { 1476 dqc, err := newDiskQuotaCacheLocal(c, c.storageRoot, c.mode) 1477 if err != nil { 1478 return err 1479 } 1480 if c.diskQuotaCache != nil { 1481 c.diskQuotaCache.Shutdown(context.TODO()) 1482 } 1483 c.diskQuotaCache = dqc 1484 return nil 1485 } 1486 1487 // MakeDiskQuotaCacheIfNotExists implements the Config interface for 1488 // ConfigLocal. 1489 func (c *ConfigLocal) MakeDiskQuotaCacheIfNotExists() error { 1490 c.lock.Lock() 1491 defer c.lock.Unlock() 1492 if c.diskQuotaCache != nil { 1493 return nil 1494 } 1495 return c.resetDiskQuotaCacheLocked() 1496 } 1497 1498 // MakeBlockMetadataStoreIfNotExists implements the Config interface for 1499 // ConfigLocal. If error happens, a Noop one is populated. 1500 func (c *ConfigLocal) MakeBlockMetadataStoreIfNotExists() (err error) { 1501 c.lock.Lock() 1502 defer c.lock.Unlock() 1503 defer func() { 1504 c.xattrStore = NewXattrStoreFromBlockMetadataStore(c.blockMetadataStore) 1505 }() 1506 if c.blockMetadataStore != nil { 1507 return nil 1508 } 1509 c.blockMetadataStore, err = newDiskBlockMetadataStore( 1510 c, c.mode, c.storageRoot) 1511 if err != nil { 1512 // TODO (KBFS-3659): when we can open levelDB read-only, 1513 // do that instead of returning a Noop version. 1514 c.blockMetadataStore = &NoopBlockMetadataStore{} 1515 return err 1516 } 1517 return nil 1518 } 1519 1520 func (c *ConfigLocal) openConfigLevelDB(configName string) ( 1521 *ldbutils.LevelDb, error) { 1522 dbPath := filepath.Join(c.storageRoot, configName) 1523 stor, err := storage.OpenFile(dbPath, false) 1524 if err != nil { 1525 return nil, err 1526 } 1527 return ldbutils.OpenLevelDb(stor, c.mode) 1528 } 1529 1530 func (c *ConfigLocal) loadSyncedTlfsLocked() (err error) { 1531 defer func() { 1532 ctx := context.TODO() 1533 c.MakeLogger("").CDebugf(ctx, "Loaded synced TLFs: %+v", err) 1534 if err != nil { 1535 // Should already be nil, but make it explicit just in 1536 // case, since the cleaning behavior depends on it being 1537 // nil if there has been an error. 1538 c.syncedTlfs = nil 1539 c.GetPerfLog().CDebugf( 1540 ctx, "KBFS failed to open synced TLFs database: %v", err) 1541 } else { 1542 c.GetPerfLog().CDebugf( 1543 ctx, "KBFS loaded %d synced TLFs", len(c.syncedTlfs)) 1544 } 1545 }() 1546 syncedTlfs := make(map[tlf.ID]FolderSyncConfig) 1547 syncedTlfPaths := make(map[string]bool) 1548 if c.mode.IsTestMode() { 1549 c.syncedTlfs = syncedTlfs 1550 c.syncedTlfPaths = syncedTlfPaths 1551 return nil 1552 } 1553 if c.storageRoot == "" { 1554 return errors.New("empty storageRoot specified for non-test run") 1555 } 1556 ldb, err := c.openConfigLevelDB(syncedTlfConfigFolderName) 1557 if err != nil { 1558 return err 1559 } 1560 defer ldb.Close() 1561 iter := ldb.NewIterator(nil, nil) 1562 defer iter.Release() 1563 1564 log := c.MakeLogger("") 1565 // If there are any un-parseable IDs, delete them. 1566 deleteBatch := new(leveldb.Batch) 1567 for iter.Next() { 1568 key := string(iter.Key()) 1569 tlfID, err := tlf.ParseID(key) 1570 if err != nil { 1571 log.Debug("deleting TLF %s from synced TLF list", key) 1572 deleteBatch.Delete(iter.Key()) 1573 continue 1574 } 1575 var config FolderSyncConfig 1576 val := iter.Value() 1577 if val != nil { 1578 err = c.codec.Decode(val, &config) 1579 if err != nil { 1580 return err 1581 } 1582 } else { 1583 // For backwards-compatibility, consider a nil value to 1584 // mean "enabled". 1585 config.Mode = keybase1.FolderSyncMode_ENABLED 1586 } 1587 syncedTlfs[tlfID] = config 1588 if config.TlfPath != "" { 1589 syncedTlfPaths[config.TlfPath] = true 1590 } 1591 } 1592 c.syncedTlfs = syncedTlfs 1593 c.syncedTlfPaths = syncedTlfPaths 1594 return ldb.Write(deleteBatch, nil) 1595 } 1596 1597 // GetTlfSyncState implements the syncedTlfGetterSetter interface for 1598 // ConfigLocal. 1599 func (c *ConfigLocal) GetTlfSyncState(tlfID tlf.ID) FolderSyncConfig { 1600 c.lock.RLock() 1601 defer c.lock.RUnlock() 1602 return c.syncedTlfs[tlfID] 1603 } 1604 1605 // IsSyncedTlf implements the syncedTlfGetterSetter interface for 1606 // ConfigLocal. 1607 func (c *ConfigLocal) IsSyncedTlf(tlfID tlf.ID) bool { 1608 return c.GetTlfSyncState(tlfID).Mode == keybase1.FolderSyncMode_ENABLED 1609 } 1610 1611 // IsSyncedTlfPath implements the syncedTlfGetterSetter interface for 1612 // ConfigLocal. 1613 func (c *ConfigLocal) IsSyncedTlfPath(tlfPath string) bool { 1614 c.lock.RLock() 1615 defer c.lock.RUnlock() 1616 return c.syncedTlfPaths[tlfPath] 1617 } 1618 1619 // OfflineAvailabilityForPath implements the offlineStatusGetter 1620 // interface for ConfigLocal. 1621 func (c *ConfigLocal) OfflineAvailabilityForPath( 1622 tlfPath string) keybase1.OfflineAvailability { 1623 if c.IsSyncedTlfPath(tlfPath) { 1624 return keybase1.OfflineAvailability_BEST_EFFORT 1625 } 1626 return keybase1.OfflineAvailability_NONE 1627 } 1628 1629 // OfflineAvailabilityForID implements the offlineStatusGetter 1630 // interface for ConfigLocal. 1631 func (c *ConfigLocal) OfflineAvailabilityForID( 1632 tlfID tlf.ID) keybase1.OfflineAvailability { 1633 if c.GetTlfSyncState(tlfID).Mode != keybase1.FolderSyncMode_DISABLED { 1634 return keybase1.OfflineAvailability_BEST_EFFORT 1635 } 1636 return keybase1.OfflineAvailability_NONE 1637 } 1638 1639 func (c *ConfigLocal) setTlfSyncState( 1640 ctx context.Context, tlfID tlf.ID, config FolderSyncConfig) ( 1641 <-chan error, error) { 1642 c.lock.Lock() 1643 defer c.lock.Unlock() 1644 diskCacheWrapped, ok := c.diskBlockCache.(*diskBlockCacheWrapped) 1645 if !ok { 1646 return nil, errors.Errorf( 1647 "invalid disk cache type to set TLF sync state: %T", 1648 c.diskBlockCache) 1649 } 1650 if !diskCacheWrapped.IsSyncCacheEnabled() { 1651 return nil, errors.New("sync block cache is not enabled") 1652 } 1653 1654 if !c.mode.IsTestMode() { 1655 if c.storageRoot == "" { 1656 return nil, errors.New( 1657 "empty storageRoot specified for non-test run") 1658 } 1659 ldb, err := c.openConfigLevelDB(syncedTlfConfigFolderName) 1660 if err != nil { 1661 return nil, err 1662 } 1663 defer ldb.Close() 1664 tlfBytes, err := tlfID.MarshalText() 1665 if err != nil { 1666 return nil, err 1667 } 1668 if config.Mode == keybase1.FolderSyncMode_DISABLED { 1669 err = ldb.Delete(tlfBytes, nil) 1670 } else { 1671 if cancel, ok := c.tlfClearCancels[tlfID]; ok { 1672 cancel() 1673 } 1674 var buf []byte 1675 buf, err = c.codec.Encode(&config) 1676 if err != nil { 1677 return nil, err 1678 } 1679 err = ldb.Put(tlfBytes, buf, nil) 1680 } 1681 if err != nil { 1682 return nil, err 1683 } 1684 } 1685 1686 ch := make(chan error, 1) 1687 if config.Mode == keybase1.FolderSyncMode_DISABLED { 1688 c.GetPerfLog().CDebugf( 1689 ctx, "Clearing KBFS sync blocks for disabled TLF %s", tlfID) 1690 c.cleanSyncBlockCacheForTlfInBackgroundLocked(tlfID, ch) 1691 } else { 1692 ch <- nil 1693 } 1694 1695 oldConfig := c.syncedTlfs[tlfID] 1696 if config.TlfPath != oldConfig.TlfPath { 1697 delete(c.syncedTlfPaths, oldConfig.TlfPath) 1698 } 1699 1700 c.syncedTlfs[tlfID] = config 1701 if config.TlfPath != "" { 1702 c.syncedTlfPaths[config.TlfPath] = true 1703 } 1704 return ch, nil 1705 } 1706 1707 // SetTlfSyncState implements the syncedTlfGetterSetter interface for 1708 // ConfigLocal. 1709 func (c *ConfigLocal) SetTlfSyncState( 1710 ctx context.Context, tlfID tlf.ID, config FolderSyncConfig) ( 1711 <-chan error, error) { 1712 if !c.IsTestMode() && config.Mode != keybase1.FolderSyncMode_ENABLED { 1713 // If we're disabling, or just changing the partial sync 1714 // config (which may be removing paths), we should cancel all 1715 // the previous prefetches for this TLF. For partial syncs, a 1716 // new sync will be started. 1717 err := c.BlockOps().Prefetcher().CancelTlfPrefetches(ctx, tlfID) 1718 if err != nil { 1719 return nil, err 1720 } 1721 } 1722 return c.setTlfSyncState(ctx, tlfID, config) 1723 } 1724 1725 // GetAllSyncedTlfs implements the syncedTlfGetterSetter interface for 1726 // ConfigLocal. 1727 func (c *ConfigLocal) GetAllSyncedTlfs() []tlf.ID { 1728 c.lock.RLock() 1729 defer c.lock.RUnlock() 1730 tlfs := make([]tlf.ID, 0, len(c.syncedTlfs)) 1731 for tlf, config := range c.syncedTlfs { 1732 if config.Mode != keybase1.FolderSyncMode_DISABLED { 1733 tlfs = append(tlfs, tlf) 1734 } 1735 } 1736 return tlfs 1737 } 1738 1739 // PrefetchStatus implements the Config interface for ConfigLocal. 1740 func (c *ConfigLocal) PrefetchStatus(ctx context.Context, tlfID tlf.ID, 1741 ptr data.BlockPointer) PrefetchStatus { 1742 dbc := c.DiskBlockCache() 1743 if dbc == nil { 1744 // We must be in testing mode, so check the block retrieval queue. 1745 bops, ok := c.BlockOps().(*BlockOpsStandard) 1746 if !ok { 1747 return NoPrefetch 1748 } 1749 status, err := bops.queue.getPrefetchStatus(ptr.ID) 1750 if err != nil { 1751 c.MakeLogger("").CDebugf(ctx, 1752 "Error getting prefetch status: %+v", err) 1753 return NoPrefetch 1754 } 1755 return status 1756 } 1757 1758 prefetchStatus, err := dbc.GetPrefetchStatus( 1759 ctx, tlfID, ptr.ID, DiskBlockAnyCache) 1760 if err != nil { 1761 return NoPrefetch 1762 } 1763 return prefetchStatus 1764 } 1765 1766 // GetRekeyFSMLimiter implements the Config interface for ConfigLocal. 1767 func (c *ConfigLocal) GetRekeyFSMLimiter() *OngoingWorkLimiter { 1768 return c.rekeyFSMLimiter 1769 } 1770 1771 // GetConflictResolutionDB implements the Config interface for ConfigLocal. 1772 func (c *ConfigLocal) GetConflictResolutionDB() (db *ldbutils.LevelDb) { 1773 return c.conflictResolutionDB 1774 } 1775 1776 // GetSettingsDB implements the Config interface for ConfigLocal. 1777 func (c *ConfigLocal) GetSettingsDB() (db *SettingsDB) { 1778 return c.settingsDB 1779 } 1780 1781 // SetKBFSService sets the KBFSService for this ConfigLocal. 1782 func (c *ConfigLocal) SetKBFSService(k *KBFSService) { 1783 c.lock.Lock() 1784 defer c.lock.Unlock() 1785 if c.kbfsService != nil { 1786 c.kbfsService.Shutdown() 1787 } 1788 c.kbfsService = k 1789 } 1790 1791 // RootNodeWrappers implements the Config interface for ConfigLocal. 1792 func (c *ConfigLocal) RootNodeWrappers() []func(Node) Node { 1793 c.lock.RLock() 1794 defer c.lock.RUnlock() 1795 return c.rootNodeWrappers 1796 } 1797 1798 // AddRootNodeWrapper implements the Config interface for ConfigLocal. 1799 func (c *ConfigLocal) AddRootNodeWrapper(f func(Node) Node) { 1800 c.lock.Lock() 1801 defer c.lock.Unlock() 1802 c.rootNodeWrappers = append(c.rootNodeWrappers, f) 1803 if c.kbfs != nil { 1804 c.kbfs.AddRootNodeWrapper(f) 1805 } 1806 } 1807 1808 // SetVLogLevel implements the Config interface for ConfigLocal. 1809 func (c *ConfigLocal) SetVLogLevel(levelString string) { 1810 c.lock.Lock() 1811 defer c.lock.Unlock() 1812 c.vdebugSetting = levelString 1813 for _, vlog := range c.vlogs { 1814 vlog.Configure(levelString) 1815 } 1816 } 1817 1818 // VLogLevel implements the Config interface for ConfigLocal. 1819 func (c *ConfigLocal) VLogLevel() string { 1820 c.lock.RLock() 1821 defer c.lock.RUnlock() 1822 return c.vdebugSetting 1823 } 1824 1825 // SetDiskCacheMode sets the disk cache mode for this config, after 1826 // construction. Mostly useful for tests. 1827 func (c *ConfigLocal) SetDiskCacheMode(m DiskCacheMode) { 1828 c.lock.Lock() 1829 defer c.lock.Unlock() 1830 c.diskCacheMode = m 1831 if c.diskCacheMode == DiskCacheModeLocal { 1832 // Any error is logged in the function itself. 1833 _ = c.loadSyncedTlfsLocked() 1834 } 1835 } 1836 1837 // SubscriptionManager implements the Config interface. 1838 func (c *ConfigLocal) SubscriptionManager( 1839 clientID SubscriptionManagerClientID, purgeable bool, 1840 notifier SubscriptionNotifier) SubscriptionManager { 1841 return c.subscriptionManagerManager.get(clientID, purgeable, notifier) 1842 } 1843 1844 // SubscriptionManagerPublisher implements the Config interface. 1845 func (c *ConfigLocal) SubscriptionManagerPublisher() SubscriptionManagerPublisher { 1846 return c.subscriptionManagerManager 1847 } 1848 1849 // KbEnv implements the Config interface. 1850 func (c *ConfigLocal) KbEnv() *libkb.Env { 1851 c.lock.RLock() 1852 defer c.lock.RUnlock() 1853 return c.kbCtx.GetEnv() 1854 } 1855 1856 // KbContext implements the Config interface. 1857 func (c *ConfigLocal) KbContext() Context { 1858 c.lock.RLock() 1859 defer c.lock.RUnlock() 1860 return c.kbCtx 1861 }