github.com/keltia/go-ipfs@v0.3.8-0.20150909044612-210793031c63/repo/fsrepo/fsrepo.go (about)

     1  package fsrepo
     2  
     3  import (
     4  	"errors"
     5  	"fmt"
     6  	"io"
     7  	"os"
     8  	"path"
     9  	"strconv"
    10  	"strings"
    11  	"sync"
    12  
    13  	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
    14  	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs"
    15  	levelds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/leveldb"
    16  	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/measure"
    17  	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/mount"
    18  	ldbopts "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
    19  	repo "github.com/ipfs/go-ipfs/repo"
    20  	"github.com/ipfs/go-ipfs/repo/common"
    21  	config "github.com/ipfs/go-ipfs/repo/config"
    22  	lockfile "github.com/ipfs/go-ipfs/repo/fsrepo/lock"
    23  	mfsr "github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
    24  	serialize "github.com/ipfs/go-ipfs/repo/fsrepo/serialize"
    25  	dir "github.com/ipfs/go-ipfs/thirdparty/dir"
    26  	"github.com/ipfs/go-ipfs/thirdparty/eventlog"
    27  	u "github.com/ipfs/go-ipfs/util"
    28  	util "github.com/ipfs/go-ipfs/util"
    29  	ds2 "github.com/ipfs/go-ipfs/util/datastore2"
    30  )
    31  
    32  // version number that we are currently expecting to see
    33  var RepoVersion = "2"
    34  
    35  var migrationInstructions = `See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md
    36  Sorry for the inconvenience. In the future, these will run automatically.`
    37  
    38  var errIncorrectRepoFmt = `Repo has incorrect version: %s
    39  Program version is: %s
    40  Please run the ipfs migration tool before continuing.
    41  ` + migrationInstructions
    42  
    43  var (
    44  	ErrNoVersion = errors.New("no version file found, please run 0-to-1 migration tool.\n" + migrationInstructions)
    45  	ErrOldRepo   = errors.New("ipfs repo found in old '~/.go-ipfs' location, please run migration tool.\n" + migrationInstructions)
    46  )
    47  
    48  type NoRepoError struct {
    49  	Path string
    50  }
    51  
    52  var _ error = NoRepoError{}
    53  
    54  func (err NoRepoError) Error() string {
    55  	return fmt.Sprintf("no ipfs repo found in %s.\nplease run: ipfs init", err.Path)
    56  }
    57  
    58  const (
    59  	leveldbDirectory = "datastore"
    60  	flatfsDirectory  = "blocks"
    61  	apiFile          = "api"
    62  )
    63  
    64  var (
    65  
    66  	// packageLock must be held to while performing any operation that modifies an
    67  	// FSRepo's state field. This includes Init, Open, Close, and Remove.
    68  	packageLock sync.Mutex
    69  
    70  	// onlyOne keeps track of open FSRepo instances.
    71  	//
    72  	// TODO: once command Context / Repo integration is cleaned up,
    73  	// this can be removed. Right now, this makes ConfigCmd.Run
    74  	// function try to open the repo twice:
    75  	//
    76  	//     $ ipfs daemon &
    77  	//     $ ipfs config foo
    78  	//
    79  	// The reason for the above is that in standalone mode without the
    80  	// daemon, `ipfs config` tries to save work by not building the
    81  	// full IpfsNode, but accessing the Repo directly.
    82  	onlyOne repo.OnlyOne
    83  )
    84  
    85  // FSRepo represents an IPFS FileSystem Repo. It is safe for use by multiple
    86  // callers.
    87  type FSRepo struct {
    88  	// has Close been called already
    89  	closed bool
    90  	// path is the file-system path
    91  	path string
    92  	// lockfile is the file system lock to prevent others from opening
    93  	// the same fsrepo path concurrently
    94  	lockfile io.Closer
    95  	config   *config.Config
    96  	ds       ds.ThreadSafeDatastore
    97  	// tracked separately for use in Close; do not use directly.
    98  	leveldbDS      levelds.Datastore
    99  	metricsBlocks  measure.DatastoreCloser
   100  	metricsLevelDB measure.DatastoreCloser
   101  }
   102  
   103  var _ repo.Repo = (*FSRepo)(nil)
   104  
   105  // Open the FSRepo at path. Returns an error if the repo is not
   106  // initialized.
   107  func Open(repoPath string) (repo.Repo, error) {
   108  	fn := func() (repo.Repo, error) {
   109  		return open(repoPath)
   110  	}
   111  	return onlyOne.Open(repoPath, fn)
   112  }
   113  
   114  func open(repoPath string) (repo.Repo, error) {
   115  	packageLock.Lock()
   116  	defer packageLock.Unlock()
   117  
   118  	r, err := newFSRepo(repoPath)
   119  	if err != nil {
   120  		return nil, err
   121  	}
   122  
   123  	// Check if its initialized
   124  	if err := checkInitialized(r.path); err != nil {
   125  		return nil, err
   126  	}
   127  
   128  	r.lockfile, err = lockfile.Lock(r.path)
   129  	if err != nil {
   130  		return nil, err
   131  	}
   132  	keepLocked := false
   133  	defer func() {
   134  		// unlock on error, leave it locked on success
   135  		if !keepLocked {
   136  			r.lockfile.Close()
   137  		}
   138  	}()
   139  
   140  	// Check version, and error out if not matching
   141  	ver, err := mfsr.RepoPath(r.path).Version()
   142  	if err != nil {
   143  		if os.IsNotExist(err) {
   144  			return nil, ErrNoVersion
   145  		}
   146  		return nil, err
   147  	}
   148  
   149  	if ver != RepoVersion {
   150  		return nil, fmt.Errorf(errIncorrectRepoFmt, ver, RepoVersion)
   151  	}
   152  
   153  	// check repo path, then check all constituent parts.
   154  	if err := dir.Writable(r.path); err != nil {
   155  		return nil, err
   156  	}
   157  
   158  	if err := r.openConfig(); err != nil {
   159  		return nil, err
   160  	}
   161  
   162  	if err := r.openDatastore(); err != nil {
   163  		return nil, err
   164  	}
   165  
   166  	// setup eventlogger
   167  	configureEventLoggerAtRepoPath(r.config, r.path)
   168  
   169  	keepLocked = true
   170  	return r, nil
   171  }
   172  
   173  func newFSRepo(rpath string) (*FSRepo, error) {
   174  	expPath, err := u.TildeExpansion(path.Clean(rpath))
   175  	if err != nil {
   176  		return nil, err
   177  	}
   178  
   179  	return &FSRepo{path: expPath}, nil
   180  }
   181  
   182  func checkInitialized(path string) error {
   183  	if !isInitializedUnsynced(path) {
   184  		alt := strings.Replace(path, ".ipfs", ".go-ipfs", 1)
   185  		if isInitializedUnsynced(alt) {
   186  			return ErrOldRepo
   187  		}
   188  		return NoRepoError{Path: path}
   189  	}
   190  	return nil
   191  }
   192  
   193  // ConfigAt returns an error if the FSRepo at the given path is not
   194  // initialized. This function allows callers to read the config file even when
   195  // another process is running and holding the lock.
   196  func ConfigAt(repoPath string) (*config.Config, error) {
   197  
   198  	// packageLock must be held to ensure that the Read is atomic.
   199  	packageLock.Lock()
   200  	defer packageLock.Unlock()
   201  
   202  	configFilename, err := config.Filename(repoPath)
   203  	if err != nil {
   204  		return nil, err
   205  	}
   206  	return serialize.Load(configFilename)
   207  }
   208  
   209  // configIsInitialized returns true if the repo is initialized at
   210  // provided |path|.
   211  func configIsInitialized(path string) bool {
   212  	configFilename, err := config.Filename(path)
   213  	if err != nil {
   214  		return false
   215  	}
   216  	if !util.FileExists(configFilename) {
   217  		return false
   218  	}
   219  	return true
   220  }
   221  
   222  func initConfig(path string, conf *config.Config) error {
   223  	if configIsInitialized(path) {
   224  		return nil
   225  	}
   226  	configFilename, err := config.Filename(path)
   227  	if err != nil {
   228  		return err
   229  	}
   230  	// initialization is the one time when it's okay to write to the config
   231  	// without reading the config from disk and merging any user-provided keys
   232  	// that may exist.
   233  	if err := serialize.WriteConfigFile(configFilename, conf); err != nil {
   234  		return err
   235  	}
   236  	return nil
   237  }
   238  
   239  // Init initializes a new FSRepo at the given path with the provided config.
   240  // TODO add support for custom datastores.
   241  func Init(repoPath string, conf *config.Config) error {
   242  
   243  	// packageLock must be held to ensure that the repo is not initialized more
   244  	// than once.
   245  	packageLock.Lock()
   246  	defer packageLock.Unlock()
   247  
   248  	if isInitializedUnsynced(repoPath) {
   249  		return nil
   250  	}
   251  
   252  	if err := initConfig(repoPath, conf); err != nil {
   253  		return err
   254  	}
   255  
   256  	// The actual datastore contents are initialized lazily when Opened.
   257  	// During Init, we merely check that the directory is writeable.
   258  	leveldbPath := path.Join(repoPath, leveldbDirectory)
   259  	if err := dir.Writable(leveldbPath); err != nil {
   260  		return fmt.Errorf("datastore: %s", err)
   261  	}
   262  
   263  	flatfsPath := path.Join(repoPath, flatfsDirectory)
   264  	if err := dir.Writable(flatfsPath); err != nil {
   265  		return fmt.Errorf("datastore: %s", err)
   266  	}
   267  
   268  	if err := dir.Writable(path.Join(repoPath, "logs")); err != nil {
   269  		return err
   270  	}
   271  
   272  	if err := mfsr.RepoPath(repoPath).WriteVersion(RepoVersion); err != nil {
   273  		return err
   274  	}
   275  
   276  	return nil
   277  }
   278  
   279  // Remove recursively removes the FSRepo at |path|.
   280  func Remove(repoPath string) error {
   281  	repoPath = path.Clean(repoPath)
   282  	return os.RemoveAll(repoPath)
   283  }
   284  
   285  // LockedByOtherProcess returns true if the FSRepo is locked by another
   286  // process. If true, then the repo cannot be opened by this process.
   287  func LockedByOtherProcess(repoPath string) (bool, error) {
   288  	repoPath = path.Clean(repoPath)
   289  	// NB: the lock is only held when repos are Open
   290  	return lockfile.Locked(repoPath)
   291  }
   292  
   293  // APIAddr returns the registered API addr, according to the api file
   294  // in the fsrepo. This is a concurrent operation, meaning that any
   295  // process may read this file. modifying this file, therefore, should
   296  // use "mv" to replace the whole file and avoid interleaved read/writes.
   297  func APIAddr(repoPath string) (string, error) {
   298  	repoPath = path.Clean(repoPath)
   299  	apiFilePath := path.Join(repoPath, apiFile)
   300  
   301  	// if there is no file, assume there is no api addr.
   302  	f, err := os.Open(apiFilePath)
   303  	if err != nil {
   304  		if os.IsNotExist(err) {
   305  			return "", repo.ErrApiNotRunning
   306  		}
   307  		return "", err
   308  	}
   309  	defer f.Close()
   310  
   311  	// read up to 2048 bytes. io.ReadAll is a vulnerability, as
   312  	// someone could hose the process by putting a massive file there.
   313  	buf := make([]byte, 2048)
   314  	n, err := f.Read(buf)
   315  	if err != nil && err != io.EOF {
   316  		return "", err
   317  	}
   318  
   319  	s := string(buf[:n])
   320  	s = strings.TrimSpace(s)
   321  	return s, nil
   322  }
   323  
   324  // SetAPIAddr writes the API Addr to the /api file.
   325  func (r *FSRepo) SetAPIAddr(addr string) error {
   326  	f, err := os.Create(path.Join(r.path, apiFile))
   327  	if err != nil {
   328  		return err
   329  	}
   330  	defer f.Close()
   331  
   332  	_, err = f.WriteString(addr)
   333  	return err
   334  }
   335  
   336  // openConfig returns an error if the config file is not present.
   337  func (r *FSRepo) openConfig() error {
   338  	configFilename, err := config.Filename(r.path)
   339  	if err != nil {
   340  		return err
   341  	}
   342  	conf, err := serialize.Load(configFilename)
   343  	if err != nil {
   344  		return err
   345  	}
   346  	r.config = conf
   347  	return nil
   348  }
   349  
   350  // openDatastore returns an error if the config file is not present.
   351  func (r *FSRepo) openDatastore() error {
   352  	leveldbPath := path.Join(r.path, leveldbDirectory)
   353  	var err error
   354  	// save leveldb reference so it can be neatly closed afterward
   355  	r.leveldbDS, err = levelds.NewDatastore(leveldbPath, &levelds.Options{
   356  		Compression: ldbopts.NoCompression,
   357  	})
   358  	if err != nil {
   359  		return errors.New("unable to open leveldb datastore")
   360  	}
   361  
   362  	// 4TB of 256kB objects ~=17M objects, splitting that 256-way
   363  	// leads to ~66k objects per dir, splitting 256*256-way leads to
   364  	// only 256.
   365  	//
   366  	// The keys seen by the block store have predictable prefixes,
   367  	// including "/" from datastore.Key and 2 bytes from multihash. To
   368  	// reach a uniform 256-way split, we need approximately 4 bytes of
   369  	// prefix.
   370  	blocksDS, err := flatfs.New(path.Join(r.path, flatfsDirectory), 4)
   371  	if err != nil {
   372  		return errors.New("unable to open flatfs datastore")
   373  	}
   374  
   375  	// Add our PeerID to metrics paths to keep them unique
   376  	//
   377  	// As some tests just pass a zero-value Config to fsrepo.Init,
   378  	// cope with missing PeerID.
   379  	id := r.config.Identity.PeerID
   380  	if id == "" {
   381  		// the tests pass in a zero Config; cope with it
   382  		id = fmt.Sprintf("uninitialized_%p", r)
   383  	}
   384  	prefix := "fsrepo." + id + ".datastore."
   385  	r.metricsBlocks = measure.New(prefix+"blocks", blocksDS)
   386  	r.metricsLevelDB = measure.New(prefix+"leveldb", r.leveldbDS)
   387  	mountDS := mount.New([]mount.Mount{
   388  		{
   389  			Prefix:    ds.NewKey("/blocks"),
   390  			Datastore: r.metricsBlocks,
   391  		},
   392  		{
   393  			Prefix:    ds.NewKey("/"),
   394  			Datastore: r.metricsLevelDB,
   395  		},
   396  	})
   397  	// Make sure it's ok to claim the virtual datastore from mount as
   398  	// threadsafe. There's no clean way to make mount itself provide
   399  	// this information without copy-pasting the code into two
   400  	// variants. This is the same dilemma as the `[].byte` attempt at
   401  	// introducing const types to Go.
   402  	var _ ds.ThreadSafeDatastore = blocksDS
   403  	var _ ds.ThreadSafeDatastore = r.leveldbDS
   404  	r.ds = ds2.ClaimThreadSafe{mountDS}
   405  	return nil
   406  }
   407  
   408  func configureEventLoggerAtRepoPath(c *config.Config, repoPath string) {
   409  	eventlog.Configure(eventlog.LevelInfo)
   410  	eventlog.Configure(eventlog.LdJSONFormatter)
   411  	eventlog.Configure(eventlog.Output(eventlog.WriterGroup))
   412  }
   413  
   414  // Close closes the FSRepo, releasing held resources.
   415  func (r *FSRepo) Close() error {
   416  	packageLock.Lock()
   417  	defer packageLock.Unlock()
   418  
   419  	if r.closed {
   420  		return errors.New("repo is closed")
   421  	}
   422  
   423  	if err := r.metricsBlocks.Close(); err != nil {
   424  		return err
   425  	}
   426  	if err := r.metricsLevelDB.Close(); err != nil {
   427  		return err
   428  	}
   429  	if err := r.leveldbDS.Close(); err != nil {
   430  		return err
   431  	}
   432  
   433  	// This code existed in the previous versions, but
   434  	// EventlogComponent.Close was never called. Preserving here
   435  	// pending further discussion.
   436  	//
   437  	// TODO It isn't part of the current contract, but callers may like for us
   438  	// to disable logging once the component is closed.
   439  	// eventlog.Configure(eventlog.Output(os.Stderr))
   440  
   441  	r.closed = true
   442  	if err := r.lockfile.Close(); err != nil {
   443  		return err
   444  	}
   445  	return nil
   446  }
   447  
   448  // Result when not Open is undefined. The method may panic if it pleases.
   449  func (r *FSRepo) Config() (*config.Config, error) {
   450  
   451  	// It is not necessary to hold the package lock since the repo is in an
   452  	// opened state. The package lock is _not_ meant to ensure that the repo is
   453  	// thread-safe. The package lock is only meant to guard againt removal and
   454  	// coordinate the lockfile. However, we provide thread-safety to keep
   455  	// things simple.
   456  	packageLock.Lock()
   457  	defer packageLock.Unlock()
   458  
   459  	if r.closed {
   460  		return nil, errors.New("cannot access config, repo not open")
   461  	}
   462  	return r.config, nil
   463  }
   464  
   465  // setConfigUnsynced is for private use.
   466  func (r *FSRepo) setConfigUnsynced(updated *config.Config) error {
   467  	configFilename, err := config.Filename(r.path)
   468  	if err != nil {
   469  		return err
   470  	}
   471  	// to avoid clobbering user-provided keys, must read the config from disk
   472  	// as a map, write the updated struct values to the map and write the map
   473  	// to disk.
   474  	var mapconf map[string]interface{}
   475  	if err := serialize.ReadConfigFile(configFilename, &mapconf); err != nil {
   476  		return err
   477  	}
   478  	m, err := config.ToMap(updated)
   479  	if err != nil {
   480  		return err
   481  	}
   482  	for k, v := range m {
   483  		mapconf[k] = v
   484  	}
   485  	if err := serialize.WriteConfigFile(configFilename, mapconf); err != nil {
   486  		return err
   487  	}
   488  	*r.config = *updated // copy so caller cannot modify this private config
   489  	return nil
   490  }
   491  
   492  // SetConfig updates the FSRepo's config.
   493  func (r *FSRepo) SetConfig(updated *config.Config) error {
   494  
   495  	// packageLock is held to provide thread-safety.
   496  	packageLock.Lock()
   497  	defer packageLock.Unlock()
   498  
   499  	return r.setConfigUnsynced(updated)
   500  }
   501  
   502  // GetConfigKey retrieves only the value of a particular key.
   503  func (r *FSRepo) GetConfigKey(key string) (interface{}, error) {
   504  	packageLock.Lock()
   505  	defer packageLock.Unlock()
   506  
   507  	if r.closed {
   508  		return nil, errors.New("repo is closed")
   509  	}
   510  
   511  	filename, err := config.Filename(r.path)
   512  	if err != nil {
   513  		return nil, err
   514  	}
   515  	var cfg map[string]interface{}
   516  	if err := serialize.ReadConfigFile(filename, &cfg); err != nil {
   517  		return nil, err
   518  	}
   519  	return common.MapGetKV(cfg, key)
   520  }
   521  
   522  // SetConfigKey writes the value of a particular key.
   523  func (r *FSRepo) SetConfigKey(key string, value interface{}) error {
   524  	packageLock.Lock()
   525  	defer packageLock.Unlock()
   526  
   527  	if r.closed {
   528  		return errors.New("repo is closed")
   529  	}
   530  
   531  	filename, err := config.Filename(r.path)
   532  	if err != nil {
   533  		return err
   534  	}
   535  	var mapconf map[string]interface{}
   536  	if err := serialize.ReadConfigFile(filename, &mapconf); err != nil {
   537  		return err
   538  	}
   539  
   540  	// Get the type of the value associated with the key
   541  	oldValue, err := common.MapGetKV(mapconf, key)
   542  	ok := true
   543  	if err != nil {
   544  		// key-value does not exist yet
   545  		switch v := value.(type) {
   546  		case string:
   547  			value, err = strconv.ParseBool(v)
   548  			if err != nil {
   549  				value, err = strconv.Atoi(v)
   550  				if err != nil {
   551  					value, err = strconv.ParseFloat(v, 32)
   552  					if err != nil {
   553  						value = v
   554  					}
   555  				}
   556  			}
   557  		default:
   558  		}
   559  	} else {
   560  		switch oldValue.(type) {
   561  		case bool:
   562  			value, ok = value.(bool)
   563  		case int:
   564  			value, ok = value.(int)
   565  		case float32:
   566  			value, ok = value.(float32)
   567  		case string:
   568  			value, ok = value.(string)
   569  		default:
   570  			value = value
   571  		}
   572  		if !ok {
   573  			return fmt.Errorf("Wrong config type, expected %T", oldValue)
   574  		}
   575  	}
   576  
   577  	if err := common.MapSetKV(mapconf, key, value); err != nil {
   578  		return err
   579  	}
   580  
   581  	// This step doubles as to validate the map against the struct
   582  	// before serialization
   583  	conf, err := config.FromMap(mapconf)
   584  	if err != nil {
   585  		return err
   586  	}
   587  	if err := serialize.WriteConfigFile(filename, mapconf); err != nil {
   588  		return err
   589  	}
   590  	return r.setConfigUnsynced(conf) // TODO roll this into this method
   591  }
   592  
   593  // Datastore returns a repo-owned datastore. If FSRepo is Closed, return value
   594  // is undefined.
   595  func (r *FSRepo) Datastore() ds.ThreadSafeDatastore {
   596  	packageLock.Lock()
   597  	d := r.ds
   598  	packageLock.Unlock()
   599  	return d
   600  }
   601  
   602  var _ io.Closer = &FSRepo{}
   603  var _ repo.Repo = &FSRepo{}
   604  
   605  // IsInitialized returns true if the repo is initialized at provided |path|.
   606  func IsInitialized(path string) bool {
   607  	// packageLock is held to ensure that another caller doesn't attempt to
   608  	// Init or Remove the repo while this call is in progress.
   609  	packageLock.Lock()
   610  	defer packageLock.Unlock()
   611  
   612  	return isInitializedUnsynced(path)
   613  }
   614  
   615  // private methods below this point. NB: packageLock must held by caller.
   616  
   617  // isInitializedUnsynced reports whether the repo is initialized. Caller must
   618  // hold the packageLock.
   619  func isInitializedUnsynced(repoPath string) bool {
   620  	if !configIsInitialized(repoPath) {
   621  		return false
   622  	}
   623  	if !util.FileExists(path.Join(repoPath, leveldbDirectory)) {
   624  		return false
   625  	}
   626  	return true
   627  }