github.com/qri-io/qri@v0.10.1-0.20220104210721-c771715036cb/transform/startf/transform_test.go (about)

     1  package startf
     2  
     3  import (
     4  	"context"
     5  	"io/ioutil"
     6  	"net/http"
     7  	"net/http/httptest"
     8  	"testing"
     9  
    10  	"github.com/google/go-cmp/cmp"
    11  	"github.com/qri-io/dataset"
    12  	"github.com/qri-io/dataset/dsio"
    13  	"github.com/qri-io/dataset/stepfile"
    14  	"github.com/qri-io/qfs"
    15  	"github.com/qri-io/qri/base"
    16  	"github.com/qri-io/qri/repo"
    17  	repoTest "github.com/qri-io/qri/repo/test"
    18  	"github.com/qri-io/starlib"
    19  	"github.com/qri-io/starlib/testdata"
    20  	"go.starlark.net/starlark"
    21  	"go.starlark.net/starlarktest"
    22  )
    23  
    24  func scriptFile(t *testing.T, path string) qfs.File {
    25  	data, err := ioutil.ReadFile(path)
    26  	if err != nil {
    27  		t.Fatal(err)
    28  	}
    29  
    30  	return qfs.NewMemfileBytes(path, data)
    31  }
    32  
    33  func TestOpts(t *testing.T) {
    34  	o := &ExecOpts{}
    35  	SetSecrets(nil)(o)
    36  	SetSecrets(map[string]string{"a": "b"})(o)
    37  	AddQriRepo(nil)(o)
    38  
    39  	expect := &ExecOpts{
    40  		Secrets:   map[string]interface{}{"a": "b"},
    41  		ErrWriter: nil,
    42  	}
    43  
    44  	if diff := cmp.Diff(expect, o); diff != "" {
    45  		t.Errorf("result mismatch (-want +got):\n%s", diff)
    46  	}
    47  }
    48  
    49  func TestExecScript(t *testing.T) {
    50  	ctx := context.Background()
    51  	ds := &dataset.Dataset{
    52  		Transform: &dataset.Transform{},
    53  	}
    54  	ds.Transform.SetScriptFile(scriptFile(t, "testdata/tf.star"))
    55  
    56  	if err := ExecScript(ctx, ds); err != nil {
    57  		t.Fatal(err)
    58  	}
    59  	if ds.Transform == nil {
    60  		t.Error("expected transform")
    61  	}
    62  
    63  	entryReader, err := dsio.NewEntryReader(ds.Structure, ds.BodyFile())
    64  	if err != nil {
    65  		t.Fatalf("couldn't create entry reader from returned dataset & body file: %s", err.Error())
    66  	}
    67  
    68  	i := 0
    69  	dsio.EachEntry(entryReader, func(n int, x dsio.Entry, e error) error {
    70  		if e != nil {
    71  			t.Errorf("entry %d iteration error: %s", i, e.Error())
    72  		}
    73  		i++
    74  		return nil
    75  	})
    76  
    77  	if i != 3 {
    78  		t.Errorf("expected 3 entries, got: %d", i)
    79  	}
    80  	if ds.Structure.Entries != 3 {
    81  		t.Errorf("expected `ds.Structure.Entries` to be 3, got: %d", i)
    82  	}
    83  }
    84  
    85  func TestExecScript2(t *testing.T) {
    86  	ctx := context.Background()
    87  	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
    88  		w.Write([]byte(`{"foo":[["bar","baz","bat"]]}`))
    89  	}))
    90  
    91  	ds := &dataset.Dataset{
    92  		Transform: &dataset.Transform{},
    93  	}
    94  	ds.Transform.SetScriptFile(scriptFile(t, "testdata/fetch.star"))
    95  	err := ExecScript(ctx, ds, func(o *ExecOpts) {
    96  		o.Globals["test_server_url"] = starlark.String(s.URL)
    97  	})
    98  
    99  	if err != nil {
   100  		t.Error(err.Error())
   101  		return
   102  	}
   103  	if ds.Transform == nil {
   104  		t.Error("expected transform")
   105  	}
   106  }
   107  
   108  func TestExecStep(t *testing.T) {
   109  	ctx := context.Background()
   110  	ds := &dataset.Dataset{
   111  		Transform: &dataset.Transform{
   112  			Steps: []*dataset.TransformStep{
   113  				{
   114  					Name:     "transform",
   115  					Syntax:   "starlark",
   116  					Category: "transform",
   117  					Script:   "ds = dataset.latest()\nds.body = [[1,2,3]]\ndataset.commit(ds)",
   118  				},
   119  			},
   120  		},
   121  	}
   122  	// Run the single step.
   123  	stepRunner := NewStepRunner(ds)
   124  	err := stepRunner.RunStep(ctx, ds, ds.Transform.Steps[0])
   125  	if err != nil {
   126  		t.Fatal(err)
   127  	}
   128  	// Check that body was set by the transform step.
   129  	bodyfile := ds.BodyFile()
   130  	if bodyfile == nil {
   131  		t.Fatal("dataset did not have body assigned")
   132  	}
   133  	// Check that the `ds.Structure.Entries` was set
   134  	if ds.Structure == nil {
   135  		t.Fatal("dataset does not have a structure assigned")
   136  	}
   137  	if ds.Structure.Entries != 1 {
   138  		t.Errorf("`ds.Structure.Entries` not assigned correctly, expected 1, got: %d", ds.Structure.Entries)
   139  	}
   140  	data, err := ioutil.ReadAll(bodyfile)
   141  	if err != nil {
   142  		t.Fatal(err)
   143  	}
   144  	actual := string(data)
   145  	expect := "1,2,3\n"
   146  	if actual != expect {
   147  		t.Errorf("expected: %q, actual: %q", expect, actual)
   148  	}
   149  }
   150  
   151  func TestEditMeta(t *testing.T) {
   152  	ctx := context.Background()
   153  	r := testRepo(t)
   154  	ds := &dataset.Dataset{
   155  		Peername:  "peer",
   156  		Name:      "movies",
   157  		Transform: &dataset.Transform{},
   158  	}
   159  	ds.Transform.SetScriptFile(scriptFile(t, "testdata/set_meta.star"))
   160  
   161  	err := ExecScript(ctx, ds, func(o *ExecOpts) {
   162  		o.ModuleLoader = testModuleLoader(t)
   163  		o.DatasetLoader = base.NewTestDatasetLoader(r.Filesystem(), r)
   164  	})
   165  	if err != nil {
   166  		t.Fatal(err)
   167  	}
   168  	if ds.Meta.Title != "new title" {
   169  		t.Errorf("meta title was not changed")
   170  	}
   171  	if ds.Structure.Entries != 2335 {
   172  		t.Errorf("`ds.Structure.Entries` incorrect, expected 2335, got: %d", ds.Structure.Entries)
   173  	}
   174  }
   175  
   176  func TestScriptError(t *testing.T) {
   177  	ctx := context.Background()
   178  	script := `error("script error")`
   179  	scriptFile := qfs.NewMemfileBytes("tf.star", []byte(script))
   180  
   181  	ds := &dataset.Dataset{
   182  		Transform: &dataset.Transform{},
   183  	}
   184  	ds.Transform.SetScriptFile(scriptFile)
   185  	if err := ExecScript(ctx, ds); err == nil {
   186  		t.Errorf("expected script to error. got nil")
   187  	}
   188  }
   189  
   190  func TestLoadDataset(t *testing.T) {
   191  	ctx := context.Background()
   192  	r := testRepo(t)
   193  
   194  	ds := &dataset.Dataset{
   195  		Transform: &dataset.Transform{},
   196  	}
   197  	ds.Transform.SetScriptFile(scriptFile(t, "testdata/load_ds.star"))
   198  
   199  	err := ExecScript(ctx, ds, func(o *ExecOpts) {
   200  		o.Repo = r
   201  		o.ModuleLoader = testModuleLoader(t)
   202  		o.DatasetLoader = base.NewTestDatasetLoader(r.Filesystem(), r)
   203  	})
   204  	if err != nil {
   205  		t.Fatal(err)
   206  	}
   207  }
   208  
   209  func TestGetMetaNilPrev(t *testing.T) {
   210  	ctx := context.Background()
   211  	ds := &dataset.Dataset{
   212  		Transform: &dataset.Transform{},
   213  	}
   214  	ds.Transform.SetScriptFile(scriptFile(t, "testdata/meta_title.star"))
   215  	err := ExecScript(ctx, ds)
   216  	if err != nil {
   217  		t.Error(err.Error())
   218  		return
   219  	}
   220  	bodyfile := ds.BodyFile()
   221  	if bodyfile == nil {
   222  		t.Fatal("dataset did not have body assigned")
   223  	}
   224  	data, _ := ioutil.ReadAll(bodyfile)
   225  	actual := string(data)
   226  	expect := "no title\n"
   227  	if actual != expect {
   228  		t.Errorf("expected: %q, actual: %q", expect, actual)
   229  	}
   230  }
   231  
   232  func TestGetMetaWithPrev(t *testing.T) {
   233  	ctx := context.Background()
   234  	ds := &dataset.Dataset{
   235  		Meta: &dataset.Meta{
   236  			Title: "test_title",
   237  		},
   238  		Transform: &dataset.Transform{},
   239  	}
   240  	ds.Transform.SetScriptFile(scriptFile(t, "testdata/meta_title.star"))
   241  	err := ExecScript(ctx, ds)
   242  	if err != nil {
   243  		t.Error(err.Error())
   244  		return
   245  	}
   246  	bodyfile := ds.BodyFile()
   247  	if bodyfile == nil {
   248  		t.Fatal("dataset did not have body assigned")
   249  	}
   250  	data, _ := ioutil.ReadAll(bodyfile)
   251  	actual := string(data)
   252  	expect := "title: test_title\n"
   253  	if actual != expect {
   254  		t.Errorf("expected: %q, actual: %q", expect, actual)
   255  	}
   256  }
   257  
   258  func testRepo(t *testing.T) repo.Repo {
   259  	mr, err := repoTest.NewTestRepo()
   260  	if err != nil {
   261  		t.Fatal(err)
   262  	}
   263  	return mr
   264  }
   265  
   266  func testModuleLoader(t *testing.T) func(thread *starlark.Thread, module string) (dict starlark.StringDict, err error) {
   267  	assertLoader := testdata.NewLoader(nil, "")
   268  	return func(thread *starlark.Thread, module string) (dict starlark.StringDict, err error) {
   269  		starlarktest.SetReporter(thread, t)
   270  		if module == "assert.star" {
   271  			return assertLoader(thread, module)
   272  		}
   273  		return starlib.Loader(thread, module)
   274  	}
   275  }
   276  
   277  func ExecScript(ctx context.Context, ds *dataset.Dataset, opts ...func(o *ExecOpts)) error {
   278  	// Convert single-file transform scripts to steps
   279  	if len(ds.Transform.Steps) == 0 && ds.Transform.ScriptFile() != nil {
   280  		steps, err := stepfile.Read(ds.Transform.ScriptFile())
   281  		if err != nil {
   282  			return err
   283  		}
   284  		ds.Transform.Steps = steps
   285  	}
   286  
   287  	runner := NewStepRunner(ds, opts...)
   288  	for _, step := range ds.Transform.Steps {
   289  		if err := runner.RunStep(ctx, ds, step); err != nil {
   290  			return err
   291  		}
   292  	}
   293  	return nil
   294  }