github.com/filecoin-project/bacalhau@v0.3.23-0.20230228154132-45c989550ace/pkg/ipfs/car/create.go (about)

     1  package car
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"fmt"
     7  	"io"
     8  	"os"
     9  	"path"
    10  
    11  	"github.com/ipfs/go-cid"
    12  	"github.com/ipfs/go-libipfs/blocks"
    13  	"github.com/ipfs/go-unixfsnode/data/builder"
    14  	"github.com/ipld/go-car/v2"
    15  	"github.com/ipld/go-car/v2/blockstore"
    16  	dagpb "github.com/ipld/go-codec-dagpb"
    17  	"github.com/ipld/go-ipld-prime"
    18  	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
    19  	"github.com/multiformats/go-multicodec"
    20  	"github.com/multiformats/go-multihash"
    21  )
    22  
    23  // copied from https://github.com/ipld/go-car/blob/master/cmd/car/create.go
    24  
    25  func CreateCar(
    26  	ctx context.Context,
    27  	inputDirectory string,
    28  	outputFile string,
    29  	// this can be 1 or 2
    30  	version int,
    31  ) (string, error) {
    32  	// make a cid with the right length that we eventually will patch with the root.
    33  	hasher, err := multihash.GetHasher(multihash.SHA2_256)
    34  	if err != nil {
    35  		return "", err
    36  	}
    37  	digest := hasher.Sum([]byte{})
    38  	hash, err := multihash.Encode(digest, multihash.SHA2_256)
    39  	if err != nil {
    40  		return "", err
    41  	}
    42  	proxyRoot := cid.NewCidV1(uint64(multicodec.DagPb), hash)
    43  
    44  	var options []car.Option
    45  
    46  	switch version {
    47  	case 1:
    48  		options = []car.Option{blockstore.WriteAsCarV1(true)}
    49  	case 2:
    50  		// already the default
    51  	default:
    52  		return "", fmt.Errorf("invalid CAR version %d", version)
    53  	}
    54  
    55  	cdest, err := blockstore.OpenReadWrite(outputFile, []cid.Cid{proxyRoot}, options...)
    56  	if err != nil {
    57  		return "", err
    58  	}
    59  
    60  	// Write the unixfs blocks into the store.
    61  	files, err := os.ReadDir(inputDirectory)
    62  	if err != nil {
    63  		return "", err
    64  	}
    65  
    66  	var carFilePaths []string
    67  	for _, file := range files {
    68  		carFilePaths = append(carFilePaths, fmt.Sprintf("%s/%s", inputDirectory, file.Name()))
    69  	}
    70  
    71  	root, err := writeFiles(ctx, cdest, carFilePaths...)
    72  	if err != nil {
    73  		return "", err
    74  	}
    75  
    76  	if err := cdest.Finalize(); err != nil {
    77  		return "", err
    78  	}
    79  	// re-open/finalize with the final root.
    80  	if err := car.ReplaceRootsInFile(outputFile, []cid.Cid{root}); err != nil {
    81  		return "", err
    82  	}
    83  	return root.String(), nil
    84  }
    85  
    86  func writeFiles(ctx context.Context, bs *blockstore.ReadWrite, paths ...string) (cid.Cid, error) {
    87  	ls := cidlink.DefaultLinkSystem()
    88  	ls.TrustedStorage = true
    89  	ls.StorageReadOpener = func(_ ipld.LinkContext, l ipld.Link) (io.Reader, error) {
    90  		cl, ok := l.(cidlink.Link)
    91  		if !ok {
    92  			return nil, fmt.Errorf("not a cidlink")
    93  		}
    94  		blk, err := bs.Get(ctx, cl.Cid)
    95  		if err != nil {
    96  			return nil, err
    97  		}
    98  		return bytes.NewBuffer(blk.RawData()), nil
    99  	}
   100  	ls.StorageWriteOpener = func(_ ipld.LinkContext) (io.Writer, ipld.BlockWriteCommitter, error) {
   101  		buf := bytes.NewBuffer(nil)
   102  		return buf, func(l ipld.Link) error {
   103  			cl, ok := l.(cidlink.Link)
   104  			if !ok {
   105  				return fmt.Errorf("not a cidlink")
   106  			}
   107  			blk, err := blocks.NewBlockWithCid(buf.Bytes(), cl.Cid)
   108  			if err != nil {
   109  				return err
   110  			}
   111  			bs.Put(ctx, blk) //nolint:errcheck
   112  			return nil
   113  		}, nil
   114  	}
   115  
   116  	topLevel := make([]dagpb.PBLink, 0, len(paths))
   117  	for _, p := range paths {
   118  		l, size, err := builder.BuildUnixFSRecursive(p, &ls)
   119  		if err != nil {
   120  			return cid.Undef, err
   121  		}
   122  		name := path.Base(p)
   123  		entry, err := builder.BuildUnixFSDirectoryEntry(name, int64(size), l)
   124  		if err != nil {
   125  			return cid.Undef, err
   126  		}
   127  		topLevel = append(topLevel, entry)
   128  	}
   129  
   130  	// make a directory for the file(s).
   131  
   132  	root, _, err := builder.BuildUnixFSDirectory(topLevel, &ls)
   133  	if err != nil {
   134  		return cid.Undef, nil
   135  	}
   136  	rcl, ok := root.(cidlink.Link)
   137  	if !ok {
   138  		return cid.Undef, fmt.Errorf("could not interpret %s", root)
   139  	}
   140  
   141  	return rcl.Cid, nil
   142  }