github.com/keltia/go-ipfs@v0.3.8-0.20150909044612-210793031c63/repo/fsrepo/fsrepo.go (about) 1 package fsrepo 2 3 import ( 4 "errors" 5 "fmt" 6 "io" 7 "os" 8 "path" 9 "strconv" 10 "strings" 11 "sync" 12 13 ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" 14 "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs" 15 levelds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/leveldb" 16 "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/measure" 17 "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/mount" 18 ldbopts "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt" 19 repo "github.com/ipfs/go-ipfs/repo" 20 "github.com/ipfs/go-ipfs/repo/common" 21 config "github.com/ipfs/go-ipfs/repo/config" 22 lockfile "github.com/ipfs/go-ipfs/repo/fsrepo/lock" 23 mfsr "github.com/ipfs/go-ipfs/repo/fsrepo/migrations" 24 serialize "github.com/ipfs/go-ipfs/repo/fsrepo/serialize" 25 dir "github.com/ipfs/go-ipfs/thirdparty/dir" 26 "github.com/ipfs/go-ipfs/thirdparty/eventlog" 27 u "github.com/ipfs/go-ipfs/util" 28 util "github.com/ipfs/go-ipfs/util" 29 ds2 "github.com/ipfs/go-ipfs/util/datastore2" 30 ) 31 32 // version number that we are currently expecting to see 33 var RepoVersion = "2" 34 35 var migrationInstructions = `See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md 36 Sorry for the inconvenience. In the future, these will run automatically.` 37 38 var errIncorrectRepoFmt = `Repo has incorrect version: %s 39 Program version is: %s 40 Please run the ipfs migration tool before continuing. 41 ` + migrationInstructions 42 43 var ( 44 ErrNoVersion = errors.New("no version file found, please run 0-to-1 migration tool.\n" + migrationInstructions) 45 ErrOldRepo = errors.New("ipfs repo found in old '~/.go-ipfs' location, please run migration tool.\n" + migrationInstructions) 46 ) 47 48 type NoRepoError struct { 49 Path string 50 } 51 52 var _ error = NoRepoError{} 53 54 func (err NoRepoError) Error() string { 55 return fmt.Sprintf("no ipfs repo found in %s.\nplease run: ipfs init", err.Path) 56 } 57 58 const ( 59 leveldbDirectory = "datastore" 60 flatfsDirectory = "blocks" 61 apiFile = "api" 62 ) 63 64 var ( 65 66 // packageLock must be held to while performing any operation that modifies an 67 // FSRepo's state field. This includes Init, Open, Close, and Remove. 68 packageLock sync.Mutex 69 70 // onlyOne keeps track of open FSRepo instances. 71 // 72 // TODO: once command Context / Repo integration is cleaned up, 73 // this can be removed. Right now, this makes ConfigCmd.Run 74 // function try to open the repo twice: 75 // 76 // $ ipfs daemon & 77 // $ ipfs config foo 78 // 79 // The reason for the above is that in standalone mode without the 80 // daemon, `ipfs config` tries to save work by not building the 81 // full IpfsNode, but accessing the Repo directly. 82 onlyOne repo.OnlyOne 83 ) 84 85 // FSRepo represents an IPFS FileSystem Repo. It is safe for use by multiple 86 // callers. 87 type FSRepo struct { 88 // has Close been called already 89 closed bool 90 // path is the file-system path 91 path string 92 // lockfile is the file system lock to prevent others from opening 93 // the same fsrepo path concurrently 94 lockfile io.Closer 95 config *config.Config 96 ds ds.ThreadSafeDatastore 97 // tracked separately for use in Close; do not use directly. 98 leveldbDS levelds.Datastore 99 metricsBlocks measure.DatastoreCloser 100 metricsLevelDB measure.DatastoreCloser 101 } 102 103 var _ repo.Repo = (*FSRepo)(nil) 104 105 // Open the FSRepo at path. Returns an error if the repo is not 106 // initialized. 107 func Open(repoPath string) (repo.Repo, error) { 108 fn := func() (repo.Repo, error) { 109 return open(repoPath) 110 } 111 return onlyOne.Open(repoPath, fn) 112 } 113 114 func open(repoPath string) (repo.Repo, error) { 115 packageLock.Lock() 116 defer packageLock.Unlock() 117 118 r, err := newFSRepo(repoPath) 119 if err != nil { 120 return nil, err 121 } 122 123 // Check if its initialized 124 if err := checkInitialized(r.path); err != nil { 125 return nil, err 126 } 127 128 r.lockfile, err = lockfile.Lock(r.path) 129 if err != nil { 130 return nil, err 131 } 132 keepLocked := false 133 defer func() { 134 // unlock on error, leave it locked on success 135 if !keepLocked { 136 r.lockfile.Close() 137 } 138 }() 139 140 // Check version, and error out if not matching 141 ver, err := mfsr.RepoPath(r.path).Version() 142 if err != nil { 143 if os.IsNotExist(err) { 144 return nil, ErrNoVersion 145 } 146 return nil, err 147 } 148 149 if ver != RepoVersion { 150 return nil, fmt.Errorf(errIncorrectRepoFmt, ver, RepoVersion) 151 } 152 153 // check repo path, then check all constituent parts. 154 if err := dir.Writable(r.path); err != nil { 155 return nil, err 156 } 157 158 if err := r.openConfig(); err != nil { 159 return nil, err 160 } 161 162 if err := r.openDatastore(); err != nil { 163 return nil, err 164 } 165 166 // setup eventlogger 167 configureEventLoggerAtRepoPath(r.config, r.path) 168 169 keepLocked = true 170 return r, nil 171 } 172 173 func newFSRepo(rpath string) (*FSRepo, error) { 174 expPath, err := u.TildeExpansion(path.Clean(rpath)) 175 if err != nil { 176 return nil, err 177 } 178 179 return &FSRepo{path: expPath}, nil 180 } 181 182 func checkInitialized(path string) error { 183 if !isInitializedUnsynced(path) { 184 alt := strings.Replace(path, ".ipfs", ".go-ipfs", 1) 185 if isInitializedUnsynced(alt) { 186 return ErrOldRepo 187 } 188 return NoRepoError{Path: path} 189 } 190 return nil 191 } 192 193 // ConfigAt returns an error if the FSRepo at the given path is not 194 // initialized. This function allows callers to read the config file even when 195 // another process is running and holding the lock. 196 func ConfigAt(repoPath string) (*config.Config, error) { 197 198 // packageLock must be held to ensure that the Read is atomic. 199 packageLock.Lock() 200 defer packageLock.Unlock() 201 202 configFilename, err := config.Filename(repoPath) 203 if err != nil { 204 return nil, err 205 } 206 return serialize.Load(configFilename) 207 } 208 209 // configIsInitialized returns true if the repo is initialized at 210 // provided |path|. 211 func configIsInitialized(path string) bool { 212 configFilename, err := config.Filename(path) 213 if err != nil { 214 return false 215 } 216 if !util.FileExists(configFilename) { 217 return false 218 } 219 return true 220 } 221 222 func initConfig(path string, conf *config.Config) error { 223 if configIsInitialized(path) { 224 return nil 225 } 226 configFilename, err := config.Filename(path) 227 if err != nil { 228 return err 229 } 230 // initialization is the one time when it's okay to write to the config 231 // without reading the config from disk and merging any user-provided keys 232 // that may exist. 233 if err := serialize.WriteConfigFile(configFilename, conf); err != nil { 234 return err 235 } 236 return nil 237 } 238 239 // Init initializes a new FSRepo at the given path with the provided config. 240 // TODO add support for custom datastores. 241 func Init(repoPath string, conf *config.Config) error { 242 243 // packageLock must be held to ensure that the repo is not initialized more 244 // than once. 245 packageLock.Lock() 246 defer packageLock.Unlock() 247 248 if isInitializedUnsynced(repoPath) { 249 return nil 250 } 251 252 if err := initConfig(repoPath, conf); err != nil { 253 return err 254 } 255 256 // The actual datastore contents are initialized lazily when Opened. 257 // During Init, we merely check that the directory is writeable. 258 leveldbPath := path.Join(repoPath, leveldbDirectory) 259 if err := dir.Writable(leveldbPath); err != nil { 260 return fmt.Errorf("datastore: %s", err) 261 } 262 263 flatfsPath := path.Join(repoPath, flatfsDirectory) 264 if err := dir.Writable(flatfsPath); err != nil { 265 return fmt.Errorf("datastore: %s", err) 266 } 267 268 if err := dir.Writable(path.Join(repoPath, "logs")); err != nil { 269 return err 270 } 271 272 if err := mfsr.RepoPath(repoPath).WriteVersion(RepoVersion); err != nil { 273 return err 274 } 275 276 return nil 277 } 278 279 // Remove recursively removes the FSRepo at |path|. 280 func Remove(repoPath string) error { 281 repoPath = path.Clean(repoPath) 282 return os.RemoveAll(repoPath) 283 } 284 285 // LockedByOtherProcess returns true if the FSRepo is locked by another 286 // process. If true, then the repo cannot be opened by this process. 287 func LockedByOtherProcess(repoPath string) (bool, error) { 288 repoPath = path.Clean(repoPath) 289 // NB: the lock is only held when repos are Open 290 return lockfile.Locked(repoPath) 291 } 292 293 // APIAddr returns the registered API addr, according to the api file 294 // in the fsrepo. This is a concurrent operation, meaning that any 295 // process may read this file. modifying this file, therefore, should 296 // use "mv" to replace the whole file and avoid interleaved read/writes. 297 func APIAddr(repoPath string) (string, error) { 298 repoPath = path.Clean(repoPath) 299 apiFilePath := path.Join(repoPath, apiFile) 300 301 // if there is no file, assume there is no api addr. 302 f, err := os.Open(apiFilePath) 303 if err != nil { 304 if os.IsNotExist(err) { 305 return "", repo.ErrApiNotRunning 306 } 307 return "", err 308 } 309 defer f.Close() 310 311 // read up to 2048 bytes. io.ReadAll is a vulnerability, as 312 // someone could hose the process by putting a massive file there. 313 buf := make([]byte, 2048) 314 n, err := f.Read(buf) 315 if err != nil && err != io.EOF { 316 return "", err 317 } 318 319 s := string(buf[:n]) 320 s = strings.TrimSpace(s) 321 return s, nil 322 } 323 324 // SetAPIAddr writes the API Addr to the /api file. 325 func (r *FSRepo) SetAPIAddr(addr string) error { 326 f, err := os.Create(path.Join(r.path, apiFile)) 327 if err != nil { 328 return err 329 } 330 defer f.Close() 331 332 _, err = f.WriteString(addr) 333 return err 334 } 335 336 // openConfig returns an error if the config file is not present. 337 func (r *FSRepo) openConfig() error { 338 configFilename, err := config.Filename(r.path) 339 if err != nil { 340 return err 341 } 342 conf, err := serialize.Load(configFilename) 343 if err != nil { 344 return err 345 } 346 r.config = conf 347 return nil 348 } 349 350 // openDatastore returns an error if the config file is not present. 351 func (r *FSRepo) openDatastore() error { 352 leveldbPath := path.Join(r.path, leveldbDirectory) 353 var err error 354 // save leveldb reference so it can be neatly closed afterward 355 r.leveldbDS, err = levelds.NewDatastore(leveldbPath, &levelds.Options{ 356 Compression: ldbopts.NoCompression, 357 }) 358 if err != nil { 359 return errors.New("unable to open leveldb datastore") 360 } 361 362 // 4TB of 256kB objects ~=17M objects, splitting that 256-way 363 // leads to ~66k objects per dir, splitting 256*256-way leads to 364 // only 256. 365 // 366 // The keys seen by the block store have predictable prefixes, 367 // including "/" from datastore.Key and 2 bytes from multihash. To 368 // reach a uniform 256-way split, we need approximately 4 bytes of 369 // prefix. 370 blocksDS, err := flatfs.New(path.Join(r.path, flatfsDirectory), 4) 371 if err != nil { 372 return errors.New("unable to open flatfs datastore") 373 } 374 375 // Add our PeerID to metrics paths to keep them unique 376 // 377 // As some tests just pass a zero-value Config to fsrepo.Init, 378 // cope with missing PeerID. 379 id := r.config.Identity.PeerID 380 if id == "" { 381 // the tests pass in a zero Config; cope with it 382 id = fmt.Sprintf("uninitialized_%p", r) 383 } 384 prefix := "fsrepo." + id + ".datastore." 385 r.metricsBlocks = measure.New(prefix+"blocks", blocksDS) 386 r.metricsLevelDB = measure.New(prefix+"leveldb", r.leveldbDS) 387 mountDS := mount.New([]mount.Mount{ 388 { 389 Prefix: ds.NewKey("/blocks"), 390 Datastore: r.metricsBlocks, 391 }, 392 { 393 Prefix: ds.NewKey("/"), 394 Datastore: r.metricsLevelDB, 395 }, 396 }) 397 // Make sure it's ok to claim the virtual datastore from mount as 398 // threadsafe. There's no clean way to make mount itself provide 399 // this information without copy-pasting the code into two 400 // variants. This is the same dilemma as the `[].byte` attempt at 401 // introducing const types to Go. 402 var _ ds.ThreadSafeDatastore = blocksDS 403 var _ ds.ThreadSafeDatastore = r.leveldbDS 404 r.ds = ds2.ClaimThreadSafe{mountDS} 405 return nil 406 } 407 408 func configureEventLoggerAtRepoPath(c *config.Config, repoPath string) { 409 eventlog.Configure(eventlog.LevelInfo) 410 eventlog.Configure(eventlog.LdJSONFormatter) 411 eventlog.Configure(eventlog.Output(eventlog.WriterGroup)) 412 } 413 414 // Close closes the FSRepo, releasing held resources. 415 func (r *FSRepo) Close() error { 416 packageLock.Lock() 417 defer packageLock.Unlock() 418 419 if r.closed { 420 return errors.New("repo is closed") 421 } 422 423 if err := r.metricsBlocks.Close(); err != nil { 424 return err 425 } 426 if err := r.metricsLevelDB.Close(); err != nil { 427 return err 428 } 429 if err := r.leveldbDS.Close(); err != nil { 430 return err 431 } 432 433 // This code existed in the previous versions, but 434 // EventlogComponent.Close was never called. Preserving here 435 // pending further discussion. 436 // 437 // TODO It isn't part of the current contract, but callers may like for us 438 // to disable logging once the component is closed. 439 // eventlog.Configure(eventlog.Output(os.Stderr)) 440 441 r.closed = true 442 if err := r.lockfile.Close(); err != nil { 443 return err 444 } 445 return nil 446 } 447 448 // Result when not Open is undefined. The method may panic if it pleases. 449 func (r *FSRepo) Config() (*config.Config, error) { 450 451 // It is not necessary to hold the package lock since the repo is in an 452 // opened state. The package lock is _not_ meant to ensure that the repo is 453 // thread-safe. The package lock is only meant to guard againt removal and 454 // coordinate the lockfile. However, we provide thread-safety to keep 455 // things simple. 456 packageLock.Lock() 457 defer packageLock.Unlock() 458 459 if r.closed { 460 return nil, errors.New("cannot access config, repo not open") 461 } 462 return r.config, nil 463 } 464 465 // setConfigUnsynced is for private use. 466 func (r *FSRepo) setConfigUnsynced(updated *config.Config) error { 467 configFilename, err := config.Filename(r.path) 468 if err != nil { 469 return err 470 } 471 // to avoid clobbering user-provided keys, must read the config from disk 472 // as a map, write the updated struct values to the map and write the map 473 // to disk. 474 var mapconf map[string]interface{} 475 if err := serialize.ReadConfigFile(configFilename, &mapconf); err != nil { 476 return err 477 } 478 m, err := config.ToMap(updated) 479 if err != nil { 480 return err 481 } 482 for k, v := range m { 483 mapconf[k] = v 484 } 485 if err := serialize.WriteConfigFile(configFilename, mapconf); err != nil { 486 return err 487 } 488 *r.config = *updated // copy so caller cannot modify this private config 489 return nil 490 } 491 492 // SetConfig updates the FSRepo's config. 493 func (r *FSRepo) SetConfig(updated *config.Config) error { 494 495 // packageLock is held to provide thread-safety. 496 packageLock.Lock() 497 defer packageLock.Unlock() 498 499 return r.setConfigUnsynced(updated) 500 } 501 502 // GetConfigKey retrieves only the value of a particular key. 503 func (r *FSRepo) GetConfigKey(key string) (interface{}, error) { 504 packageLock.Lock() 505 defer packageLock.Unlock() 506 507 if r.closed { 508 return nil, errors.New("repo is closed") 509 } 510 511 filename, err := config.Filename(r.path) 512 if err != nil { 513 return nil, err 514 } 515 var cfg map[string]interface{} 516 if err := serialize.ReadConfigFile(filename, &cfg); err != nil { 517 return nil, err 518 } 519 return common.MapGetKV(cfg, key) 520 } 521 522 // SetConfigKey writes the value of a particular key. 523 func (r *FSRepo) SetConfigKey(key string, value interface{}) error { 524 packageLock.Lock() 525 defer packageLock.Unlock() 526 527 if r.closed { 528 return errors.New("repo is closed") 529 } 530 531 filename, err := config.Filename(r.path) 532 if err != nil { 533 return err 534 } 535 var mapconf map[string]interface{} 536 if err := serialize.ReadConfigFile(filename, &mapconf); err != nil { 537 return err 538 } 539 540 // Get the type of the value associated with the key 541 oldValue, err := common.MapGetKV(mapconf, key) 542 ok := true 543 if err != nil { 544 // key-value does not exist yet 545 switch v := value.(type) { 546 case string: 547 value, err = strconv.ParseBool(v) 548 if err != nil { 549 value, err = strconv.Atoi(v) 550 if err != nil { 551 value, err = strconv.ParseFloat(v, 32) 552 if err != nil { 553 value = v 554 } 555 } 556 } 557 default: 558 } 559 } else { 560 switch oldValue.(type) { 561 case bool: 562 value, ok = value.(bool) 563 case int: 564 value, ok = value.(int) 565 case float32: 566 value, ok = value.(float32) 567 case string: 568 value, ok = value.(string) 569 default: 570 value = value 571 } 572 if !ok { 573 return fmt.Errorf("Wrong config type, expected %T", oldValue) 574 } 575 } 576 577 if err := common.MapSetKV(mapconf, key, value); err != nil { 578 return err 579 } 580 581 // This step doubles as to validate the map against the struct 582 // before serialization 583 conf, err := config.FromMap(mapconf) 584 if err != nil { 585 return err 586 } 587 if err := serialize.WriteConfigFile(filename, mapconf); err != nil { 588 return err 589 } 590 return r.setConfigUnsynced(conf) // TODO roll this into this method 591 } 592 593 // Datastore returns a repo-owned datastore. If FSRepo is Closed, return value 594 // is undefined. 595 func (r *FSRepo) Datastore() ds.ThreadSafeDatastore { 596 packageLock.Lock() 597 d := r.ds 598 packageLock.Unlock() 599 return d 600 } 601 602 var _ io.Closer = &FSRepo{} 603 var _ repo.Repo = &FSRepo{} 604 605 // IsInitialized returns true if the repo is initialized at provided |path|. 606 func IsInitialized(path string) bool { 607 // packageLock is held to ensure that another caller doesn't attempt to 608 // Init or Remove the repo while this call is in progress. 609 packageLock.Lock() 610 defer packageLock.Unlock() 611 612 return isInitializedUnsynced(path) 613 } 614 615 // private methods below this point. NB: packageLock must held by caller. 616 617 // isInitializedUnsynced reports whether the repo is initialized. Caller must 618 // hold the packageLock. 619 func isInitializedUnsynced(repoPath string) bool { 620 if !configIsInitialized(repoPath) { 621 return false 622 } 623 if !util.FileExists(path.Join(repoPath, leveldbDirectory)) { 624 return false 625 } 626 return true 627 }