github.com/keltia/go-ipfs@v0.3.8-0.20150909044612-210793031c63/importer/helpers/helpers.go (about) 1 package helpers 2 3 import ( 4 "fmt" 5 6 "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" 7 key "github.com/ipfs/go-ipfs/blocks/key" 8 chunk "github.com/ipfs/go-ipfs/importer/chunk" 9 dag "github.com/ipfs/go-ipfs/merkledag" 10 "github.com/ipfs/go-ipfs/pin" 11 ft "github.com/ipfs/go-ipfs/unixfs" 12 ) 13 14 // BlockSizeLimit specifies the maximum size an imported block can have. 15 var BlockSizeLimit = 1048576 // 1 MB 16 17 // rough estimates on expected sizes 18 var roughDataBlockSize = chunk.DefaultBlockSize 19 var roughLinkBlockSize = 1 << 13 // 8KB 20 var roughLinkSize = 34 + 8 + 5 // sha256 multihash + size + no name + protobuf framing 21 22 // DefaultLinksPerBlock governs how the importer decides how many links there 23 // will be per block. This calculation is based on expected distributions of: 24 // * the expected distribution of block sizes 25 // * the expected distribution of link sizes 26 // * desired access speed 27 // For now, we use: 28 // 29 // var roughLinkBlockSize = 1 << 13 // 8KB 30 // var roughLinkSize = 288 // sha256 + framing + name 31 // var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize) 32 // 33 // See calc_test.go 34 var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize) 35 36 // ErrSizeLimitExceeded signals that a block is larger than BlockSizeLimit. 37 var ErrSizeLimitExceeded = fmt.Errorf("object size limit exceeded") 38 39 // UnixfsNode is a struct created to aid in the generation 40 // of unixfs DAG trees 41 type UnixfsNode struct { 42 node *dag.Node 43 ufmt *ft.FSNode 44 } 45 46 // NewUnixfsNode creates a new Unixfs node to represent a file 47 func NewUnixfsNode() *UnixfsNode { 48 return &UnixfsNode{ 49 node: new(dag.Node), 50 ufmt: &ft.FSNode{Type: ft.TFile}, 51 } 52 } 53 54 // NewUnixfsBlock creates a new Unixfs node to represent a raw data block 55 func NewUnixfsBlock() *UnixfsNode { 56 return &UnixfsNode{ 57 node: new(dag.Node), 58 ufmt: &ft.FSNode{Type: ft.TRaw}, 59 } 60 } 61 62 // NewUnixfsNodeFromDag reconstructs a Unixfs node from a given dag node 63 func NewUnixfsNodeFromDag(nd *dag.Node) (*UnixfsNode, error) { 64 mb, err := ft.FSNodeFromBytes(nd.Data) 65 if err != nil { 66 return nil, err 67 } 68 69 return &UnixfsNode{ 70 node: nd, 71 ufmt: mb, 72 }, nil 73 } 74 75 func (n *UnixfsNode) NumChildren() int { 76 return n.ufmt.NumChildren() 77 } 78 79 func (n *UnixfsNode) GetChild(ctx context.Context, i int, ds dag.DAGService) (*UnixfsNode, error) { 80 nd, err := n.node.Links[i].GetNode(ctx, ds) 81 if err != nil { 82 return nil, err 83 } 84 85 return NewUnixfsNodeFromDag(nd) 86 } 87 88 // addChild will add the given UnixfsNode as a child of the receiver. 89 // the passed in DagBuilderHelper is used to store the child node an 90 // pin it locally so it doesnt get lost 91 func (n *UnixfsNode) AddChild(child *UnixfsNode, db *DagBuilderHelper) error { 92 n.ufmt.AddBlockSize(child.ufmt.FileSize()) 93 94 childnode, err := child.GetDagNode() 95 if err != nil { 96 return err 97 } 98 99 // Add a link to this node without storing a reference to the memory 100 // This way, we avoid nodes building up and consuming all of our RAM 101 err = n.node.AddNodeLinkClean("", childnode) 102 if err != nil { 103 return err 104 } 105 106 _, err = db.batch.Add(childnode) 107 if err != nil { 108 return err 109 } 110 111 // Pin the child node indirectly 112 err = db.ncb(childnode, false) 113 if err != nil { 114 return err 115 } 116 117 return nil 118 } 119 120 // Removes the child node at the given index 121 func (n *UnixfsNode) RemoveChild(index int, dbh *DagBuilderHelper) { 122 k := key.Key(n.node.Links[index].Hash) 123 if dbh.mp != nil { 124 dbh.mp.RemovePinWithMode(k, pin.Indirect) 125 } 126 n.ufmt.RemoveBlockSize(index) 127 n.node.Links = append(n.node.Links[:index], n.node.Links[index+1:]...) 128 } 129 130 func (n *UnixfsNode) SetData(data []byte) { 131 n.ufmt.Data = data 132 } 133 134 // getDagNode fills out the proper formatting for the unixfs node 135 // inside of a DAG node and returns the dag node 136 func (n *UnixfsNode) GetDagNode() (*dag.Node, error) { 137 data, err := n.ufmt.GetBytes() 138 if err != nil { 139 return nil, err 140 } 141 n.node.Data = data 142 return n.node, nil 143 }