github.com/keltia/go-ipfs@v0.3.8-0.20150909044612-210793031c63/blocks/blockstore/blockstore.go (about)

     1  // package blockstore implements a thin wrapper over a datastore, giving a
     2  // clean interface for Getting and Putting block objects.
     3  package blockstore
     4  
     5  import (
     6  	"errors"
     7  
     8  	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
     9  	dsns "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/namespace"
    10  	dsq "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
    11  	mh "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
    12  	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
    13  	blocks "github.com/ipfs/go-ipfs/blocks"
    14  	key "github.com/ipfs/go-ipfs/blocks/key"
    15  	eventlog "github.com/ipfs/go-ipfs/thirdparty/eventlog"
    16  )
    17  
    18  var log = eventlog.Logger("blockstore")
    19  
    20  // BlockPrefix namespaces blockstore datastores
    21  var BlockPrefix = ds.NewKey("blocks")
    22  
    23  var ValueTypeMismatch = errors.New("The retrieved value is not a Block")
    24  
    25  var ErrNotFound = errors.New("blockstore: block not found")
    26  
    27  // Blockstore wraps a ThreadSafeDatastore
    28  type Blockstore interface {
    29  	DeleteBlock(key.Key) error
    30  	Has(key.Key) (bool, error)
    31  	Get(key.Key) (*blocks.Block, error)
    32  	Put(*blocks.Block) error
    33  	PutMany([]*blocks.Block) error
    34  
    35  	AllKeysChan(ctx context.Context) (<-chan key.Key, error)
    36  }
    37  
    38  func NewBlockstore(d ds.ThreadSafeDatastore) Blockstore {
    39  	dd := dsns.Wrap(d, BlockPrefix)
    40  	return &blockstore{
    41  		datastore: dd,
    42  	}
    43  }
    44  
    45  type blockstore struct {
    46  	datastore ds.BatchingDatastore
    47  	// cant be ThreadSafeDatastore cause namespace.Datastore doesnt support it.
    48  	// we do check it on `NewBlockstore` though.
    49  }
    50  
    51  func (bs *blockstore) Get(k key.Key) (*blocks.Block, error) {
    52  	maybeData, err := bs.datastore.Get(k.DsKey())
    53  	if err == ds.ErrNotFound {
    54  		return nil, ErrNotFound
    55  	}
    56  	if err != nil {
    57  		return nil, err
    58  	}
    59  	bdata, ok := maybeData.([]byte)
    60  	if !ok {
    61  		return nil, ValueTypeMismatch
    62  	}
    63  
    64  	return blocks.NewBlockWithHash(bdata, mh.Multihash(k))
    65  }
    66  
    67  func (bs *blockstore) Put(block *blocks.Block) error {
    68  	k := block.Key().DsKey()
    69  
    70  	// Has is cheaper than Put, so see if we already have it
    71  	exists, err := bs.datastore.Has(k)
    72  	if err == nil && exists {
    73  		return nil // already stored.
    74  	}
    75  	return bs.datastore.Put(k, block.Data)
    76  }
    77  
    78  func (bs *blockstore) PutMany(blocks []*blocks.Block) error {
    79  	t, err := bs.datastore.Batch()
    80  	if err != nil {
    81  		return err
    82  	}
    83  	for _, b := range blocks {
    84  		k := b.Key().DsKey()
    85  		exists, err := bs.datastore.Has(k)
    86  		if err == nil && exists {
    87  			continue
    88  		}
    89  
    90  		err = t.Put(k, b.Data)
    91  		if err != nil {
    92  			return err
    93  		}
    94  	}
    95  	return t.Commit()
    96  }
    97  
    98  func (bs *blockstore) Has(k key.Key) (bool, error) {
    99  	return bs.datastore.Has(k.DsKey())
   100  }
   101  
   102  func (s *blockstore) DeleteBlock(k key.Key) error {
   103  	return s.datastore.Delete(k.DsKey())
   104  }
   105  
   106  // AllKeysChan runs a query for keys from the blockstore.
   107  // this is very simplistic, in the future, take dsq.Query as a param?
   108  //
   109  // AllKeysChan respects context
   110  func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan key.Key, error) {
   111  
   112  	// KeysOnly, because that would be _a lot_ of data.
   113  	q := dsq.Query{KeysOnly: true}
   114  	// datastore/namespace does *NOT* fix up Query.Prefix
   115  	q.Prefix = BlockPrefix.String()
   116  	res, err := bs.datastore.Query(q)
   117  	if err != nil {
   118  		return nil, err
   119  	}
   120  
   121  	// this function is here to compartmentalize
   122  	get := func() (k key.Key, ok bool) {
   123  		select {
   124  		case <-ctx.Done():
   125  			return k, false
   126  		case e, more := <-res.Next():
   127  			if !more {
   128  				return k, false
   129  			}
   130  			if e.Error != nil {
   131  				log.Debug("blockstore.AllKeysChan got err:", e.Error)
   132  				return k, false
   133  			}
   134  
   135  			// need to convert to key.Key using key.KeyFromDsKey.
   136  			k = key.KeyFromDsKey(ds.NewKey(e.Key))
   137  			log.Debug("blockstore: query got key", k)
   138  
   139  			// key must be a multihash. else ignore it.
   140  			_, err := mh.Cast([]byte(k))
   141  			if err != nil {
   142  				return "", true
   143  			}
   144  
   145  			return k, true
   146  		}
   147  	}
   148  
   149  	output := make(chan key.Key)
   150  	go func() {
   151  		defer func() {
   152  			res.Process().Close() // ensure exit (signals early exit, too)
   153  			close(output)
   154  		}()
   155  
   156  		for {
   157  			k, ok := get()
   158  			if !ok {
   159  				return
   160  			}
   161  			if k == "" {
   162  				continue
   163  			}
   164  
   165  			select {
   166  			case <-ctx.Done():
   167  				return
   168  			case output <- k:
   169  			}
   170  		}
   171  	}()
   172  
   173  	return output, nil
   174  }