github.com/keltia/go-ipfs@v0.3.8-0.20150909044612-210793031c63/importer/helpers/helpers.go (about)

     1  package helpers
     2  
     3  import (
     4  	"fmt"
     5  
     6  	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
     7  	key "github.com/ipfs/go-ipfs/blocks/key"
     8  	chunk "github.com/ipfs/go-ipfs/importer/chunk"
     9  	dag "github.com/ipfs/go-ipfs/merkledag"
    10  	"github.com/ipfs/go-ipfs/pin"
    11  	ft "github.com/ipfs/go-ipfs/unixfs"
    12  )
    13  
    14  // BlockSizeLimit specifies the maximum size an imported block can have.
    15  var BlockSizeLimit = 1048576 // 1 MB
    16  
    17  // rough estimates on expected sizes
    18  var roughDataBlockSize = chunk.DefaultBlockSize
    19  var roughLinkBlockSize = 1 << 13 // 8KB
    20  var roughLinkSize = 34 + 8 + 5   // sha256 multihash + size + no name + protobuf framing
    21  
    22  // DefaultLinksPerBlock governs how the importer decides how many links there
    23  // will be per block. This calculation is based on expected distributions of:
    24  //  * the expected distribution of block sizes
    25  //  * the expected distribution of link sizes
    26  //  * desired access speed
    27  // For now, we use:
    28  //
    29  //   var roughLinkBlockSize = 1 << 13 // 8KB
    30  //   var roughLinkSize = 288          // sha256 + framing + name
    31  //   var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize)
    32  //
    33  // See calc_test.go
    34  var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize)
    35  
    36  // ErrSizeLimitExceeded signals that a block is larger than BlockSizeLimit.
    37  var ErrSizeLimitExceeded = fmt.Errorf("object size limit exceeded")
    38  
    39  // UnixfsNode is a struct created to aid in the generation
    40  // of unixfs DAG trees
    41  type UnixfsNode struct {
    42  	node *dag.Node
    43  	ufmt *ft.FSNode
    44  }
    45  
    46  // NewUnixfsNode creates a new Unixfs node to represent a file
    47  func NewUnixfsNode() *UnixfsNode {
    48  	return &UnixfsNode{
    49  		node: new(dag.Node),
    50  		ufmt: &ft.FSNode{Type: ft.TFile},
    51  	}
    52  }
    53  
    54  // NewUnixfsBlock creates a new Unixfs node to represent a raw data block
    55  func NewUnixfsBlock() *UnixfsNode {
    56  	return &UnixfsNode{
    57  		node: new(dag.Node),
    58  		ufmt: &ft.FSNode{Type: ft.TRaw},
    59  	}
    60  }
    61  
    62  // NewUnixfsNodeFromDag reconstructs a Unixfs node from a given dag node
    63  func NewUnixfsNodeFromDag(nd *dag.Node) (*UnixfsNode, error) {
    64  	mb, err := ft.FSNodeFromBytes(nd.Data)
    65  	if err != nil {
    66  		return nil, err
    67  	}
    68  
    69  	return &UnixfsNode{
    70  		node: nd,
    71  		ufmt: mb,
    72  	}, nil
    73  }
    74  
    75  func (n *UnixfsNode) NumChildren() int {
    76  	return n.ufmt.NumChildren()
    77  }
    78  
    79  func (n *UnixfsNode) GetChild(ctx context.Context, i int, ds dag.DAGService) (*UnixfsNode, error) {
    80  	nd, err := n.node.Links[i].GetNode(ctx, ds)
    81  	if err != nil {
    82  		return nil, err
    83  	}
    84  
    85  	return NewUnixfsNodeFromDag(nd)
    86  }
    87  
    88  // addChild will add the given UnixfsNode as a child of the receiver.
    89  // the passed in DagBuilderHelper is used to store the child node an
    90  // pin it locally so it doesnt get lost
    91  func (n *UnixfsNode) AddChild(child *UnixfsNode, db *DagBuilderHelper) error {
    92  	n.ufmt.AddBlockSize(child.ufmt.FileSize())
    93  
    94  	childnode, err := child.GetDagNode()
    95  	if err != nil {
    96  		return err
    97  	}
    98  
    99  	// Add a link to this node without storing a reference to the memory
   100  	// This way, we avoid nodes building up and consuming all of our RAM
   101  	err = n.node.AddNodeLinkClean("", childnode)
   102  	if err != nil {
   103  		return err
   104  	}
   105  
   106  	_, err = db.batch.Add(childnode)
   107  	if err != nil {
   108  		return err
   109  	}
   110  
   111  	// Pin the child node indirectly
   112  	err = db.ncb(childnode, false)
   113  	if err != nil {
   114  		return err
   115  	}
   116  
   117  	return nil
   118  }
   119  
   120  // Removes the child node at the given index
   121  func (n *UnixfsNode) RemoveChild(index int, dbh *DagBuilderHelper) {
   122  	k := key.Key(n.node.Links[index].Hash)
   123  	if dbh.mp != nil {
   124  		dbh.mp.RemovePinWithMode(k, pin.Indirect)
   125  	}
   126  	n.ufmt.RemoveBlockSize(index)
   127  	n.node.Links = append(n.node.Links[:index], n.node.Links[index+1:]...)
   128  }
   129  
   130  func (n *UnixfsNode) SetData(data []byte) {
   131  	n.ufmt.Data = data
   132  }
   133  
   134  // getDagNode fills out the proper formatting for the unixfs node
   135  // inside of a DAG node and returns the dag node
   136  func (n *UnixfsNode) GetDagNode() (*dag.Node, error) {
   137  	data, err := n.ufmt.GetBytes()
   138  	if err != nil {
   139  		return nil, err
   140  	}
   141  	n.node.Data = data
   142  	return n.node, nil
   143  }