github.com/keltia/go-ipfs@v0.3.8-0.20150909044612-210793031c63/importer/importer.go (about) 1 // package importer implements utilities used to create ipfs DAGs from files 2 // and readers 3 package importer 4 5 import ( 6 "fmt" 7 "os" 8 9 "github.com/ipfs/go-ipfs/commands/files" 10 bal "github.com/ipfs/go-ipfs/importer/balanced" 11 "github.com/ipfs/go-ipfs/importer/chunk" 12 h "github.com/ipfs/go-ipfs/importer/helpers" 13 trickle "github.com/ipfs/go-ipfs/importer/trickle" 14 dag "github.com/ipfs/go-ipfs/merkledag" 15 "github.com/ipfs/go-ipfs/pin" 16 u "github.com/ipfs/go-ipfs/util" 17 ) 18 19 var log = u.Logger("importer") 20 21 // Builds a DAG from the given file, writing created blocks to disk as they are 22 // created 23 func BuildDagFromFile(fpath string, ds dag.DAGService, mp pin.ManualPinner) (*dag.Node, error) { 24 stat, err := os.Lstat(fpath) 25 if err != nil { 26 return nil, err 27 } 28 29 if stat.IsDir() { 30 return nil, fmt.Errorf("`%s` is a directory", fpath) 31 } 32 33 f, err := files.NewSerialFile(fpath, fpath, stat) 34 if err != nil { 35 return nil, err 36 } 37 defer f.Close() 38 39 return BuildDagFromReader(ds, chunk.NewSizeSplitter(f, chunk.DefaultBlockSize), BasicPinnerCB(mp)) 40 } 41 42 func BuildDagFromReader(ds dag.DAGService, spl chunk.Splitter, ncb h.NodeCB) (*dag.Node, error) { 43 // Start the splitter 44 blkch, errch := chunk.Chan(spl) 45 46 dbp := h.DagBuilderParams{ 47 Dagserv: ds, 48 Maxlinks: h.DefaultLinksPerBlock, 49 NodeCB: ncb, 50 } 51 52 return bal.BalancedLayout(dbp.New(blkch, errch)) 53 } 54 55 func BuildTrickleDagFromReader(ds dag.DAGService, spl chunk.Splitter, ncb h.NodeCB) (*dag.Node, error) { 56 // Start the splitter 57 blkch, errch := chunk.Chan(spl) 58 59 dbp := h.DagBuilderParams{ 60 Dagserv: ds, 61 Maxlinks: h.DefaultLinksPerBlock, 62 NodeCB: ncb, 63 } 64 65 return trickle.TrickleLayout(dbp.New(blkch, errch)) 66 } 67 68 func BasicPinnerCB(p pin.ManualPinner) h.NodeCB { 69 return func(n *dag.Node, last bool) error { 70 k, err := n.Key() 71 if err != nil { 72 return err 73 } 74 75 if last { 76 p.PinWithMode(k, pin.Recursive) 77 return p.Flush() 78 } else { 79 p.PinWithMode(k, pin.Indirect) 80 return nil 81 } 82 } 83 } 84 85 func PinIndirectCB(p pin.ManualPinner) h.NodeCB { 86 return func(n *dag.Node, last bool) error { 87 k, err := n.Key() 88 if err != nil { 89 return err 90 } 91 92 p.PinWithMode(k, pin.Indirect) 93 return nil 94 } 95 }