github.com/qri-io/qri@v0.10.1-0.20220104210721-c771715036cb/base/dsfs/load_test.go (about)

     1  package dsfs
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"fmt"
     7  	"io"
     8  	"io/ioutil"
     9  	"strings"
    10  	"testing"
    11  
    12  	"github.com/qri-io/dataset"
    13  	"github.com/qri-io/qfs"
    14  	testkeys "github.com/qri-io/qri/auth/key/test"
    15  	"github.com/qri-io/qri/event"
    16  )
    17  
    18  func TestLoadDataset(t *testing.T) {
    19  	ctx := context.Background()
    20  	fs := qfs.NewMemFS()
    21  	dsData, err := ioutil.ReadFile("testdata/all_fields/input.dataset.json")
    22  	if err != nil {
    23  		t.Errorf("error loading test dataset: %s", err.Error())
    24  		return
    25  	}
    26  	ds := &dataset.Dataset{}
    27  	if err := ds.UnmarshalJSON(dsData); err != nil {
    28  		t.Fatal(err)
    29  	}
    30  	body, err := ioutil.ReadFile("testdata/all_fields/body.csv")
    31  	if err != nil {
    32  		t.Fatal(err)
    33  	}
    34  
    35  	ds.SetBodyFile(qfs.NewMemfileBytes("/body.csv", body))
    36  
    37  	// These tests are using hard-coded ids that require this exact peer's private key.
    38  	pk := testkeys.GetKeyData(10).PrivKey
    39  
    40  	apath, err := WriteDataset(ctx, fs, fs, nil, ds, event.NilBus, pk, SaveSwitches{})
    41  	if err != nil {
    42  		t.Fatal(err)
    43  	}
    44  
    45  	loadedDataset, err := LoadDataset(ctx, fs, apath)
    46  	if err != nil {
    47  		t.Fatal(err)
    48  	}
    49  	// prove we aren't returning a path to a dataset that ends with `/dataset.json`
    50  	if strings.Contains(loadedDataset.Path, PackageFileDataset.Filename()) {
    51  		t.Errorf("path should not contain the basename of the dataset file: %s", loadedDataset.Path)
    52  	}
    53  
    54  	cases := []struct {
    55  		ds  *dataset.Dataset
    56  		err string
    57  	}{
    58  		{dataset.NewDatasetRef("/bad/path"),
    59  			"loading dataset: reading dataset.json file: path not found"},
    60  		{&dataset.Dataset{
    61  			Meta: dataset.NewMetaRef("QmFoo"),
    62  		}, "loading dataset: reading dataset.json file: file is not a directory"},
    63  
    64  		// TODO (b5) - fix
    65  		// {&dataset.Dataset{
    66  		// 	Structure: dataset.NewStructureRef(fmt.Sprintf("%s/bad/path", apath)),
    67  		// }, "error loading dataset structure: error loading structure file: cafs: path not found"},
    68  		// {&dataset.Dataset{
    69  		// 	Structure: dataset.NewStructureRef("/bad/path"),
    70  		// }, "error loading dataset structure: error loading structure file: cafs: path not found"},
    71  		// {&dataset.Dataset{
    72  		// 	Transform: dataset.NewTransformRef("/bad/path"),
    73  		// }, "error loading dataset transform: error loading transform raw data: cafs: path not found"},
    74  		// {&dataset.Dataset{
    75  		// 	Commit: dataset.NewCommitRef("/bad/path"),
    76  		// }, "error loading dataset commit: error loading commit file: cafs: path not found"},
    77  		// {&dataset.Dataset{
    78  		// 	Viz: dataset.NewVizRef("/bad/path"),
    79  		// }, "error loading dataset viz: error loading viz file: cafs: path not found"},
    80  	}
    81  
    82  	for i, c := range cases {
    83  		path := c.ds.Path
    84  		if !c.ds.IsEmpty() {
    85  			dsf, err := JSONFile(PackageFileDataset.String(), c.ds)
    86  			if err != nil {
    87  				t.Errorf("case %d error generating json file: %s", i, err.Error())
    88  				continue
    89  			}
    90  
    91  			path, err = fs.Put(ctx, qfs.NewMemfileReader(PackageFileDataset.String(), dsf))
    92  			if err != nil {
    93  				t.Errorf("case %d error putting file in store: %s", i, err.Error())
    94  				continue
    95  			}
    96  		}
    97  
    98  		_, err = LoadDataset(ctx, fs, path)
    99  		if !(err != nil && c.err == "" || err != nil && err.Error() == c.err) {
   100  			t.Errorf("case %d error mismatch. expected: '%s', got: '%s'", i, c.err, err)
   101  			continue
   102  		}
   103  	}
   104  }
   105  
   106  func TestLoadBody(t *testing.T) {
   107  	ctx := context.Background()
   108  	datasets, fs, err := makeFilestore()
   109  	if err != nil {
   110  		t.Fatalf("error creating test filestore: %s", err.Error())
   111  	}
   112  
   113  	ds, err := LoadDataset(ctx, fs, datasets["cities"])
   114  	if err != nil {
   115  		t.Fatalf("error loading dataset: %s", err.Error())
   116  	}
   117  
   118  	f, err := LoadBody(ctx, fs, ds)
   119  	if err != nil {
   120  		t.Fatalf("error loading data: %s", err.Error())
   121  	}
   122  
   123  	data, err := ioutil.ReadAll(f)
   124  	if err != nil {
   125  		t.Fatalf("error reading data file: %s", err.Error())
   126  	}
   127  
   128  	eq, err := ioutil.ReadFile("testdata/cities/body.csv")
   129  	if err != nil {
   130  		t.Fatalf("error reading test file: %s", err.Error())
   131  	}
   132  
   133  	if !bytes.Equal(data, eq) {
   134  		t.Errorf("byte mismatch. expected: %s, got: %s", string(eq), string(data))
   135  	}
   136  }
   137  
   138  func TestLoadTransform(t *testing.T) {
   139  	// TODO - restore
   140  	// store := cafs.NewMapstore()
   141  	// q := &dataset.AbstractTransform{Statement: "select * from whatever booooooo go home"}
   142  	// a, err := SaveAbstractTransform(store, q, true)
   143  	// if err != nil {
   144  	// 	t.Errorf(err.Error())
   145  	// 	return
   146  	// }
   147  
   148  	// if _, err = LoadTransform(store, a); err != nil {
   149  	// 	t.Errorf(err.Error())
   150  	// }
   151  	// TODO - other tests & stuff
   152  }
   153  
   154  var ErrStreamsNotEqual = fmt.Errorf("streams are not equal")
   155  
   156  // EqualReader confirms two readers are exactly the same, throwing an error
   157  // if they return
   158  type EqualReader struct {
   159  	a, b io.Reader
   160  }
   161  
   162  func (r *EqualReader) Read(p []byte) (int, error) {
   163  	pb := make([]byte, len(p))
   164  	readA, err := r.a.Read(p)
   165  	if err != nil {
   166  		return readA, err
   167  	}
   168  
   169  	readB, err := r.b.Read(pb)
   170  	if err != nil {
   171  		return readA, err
   172  	}
   173  
   174  	if readA != readB {
   175  		return readA, ErrStreamsNotEqual
   176  	}
   177  
   178  	for i, b := range p {
   179  		if pb[i] != b {
   180  			return readA, ErrStreamsNotEqual
   181  		}
   182  	}
   183  
   184  	return readA, nil
   185  }