github.com/qri-io/qri@v0.10.1-0.20220104210721-c771715036cb/transform/startf/transform_test.go (about) 1 package startf 2 3 import ( 4 "context" 5 "io/ioutil" 6 "net/http" 7 "net/http/httptest" 8 "testing" 9 10 "github.com/google/go-cmp/cmp" 11 "github.com/qri-io/dataset" 12 "github.com/qri-io/dataset/dsio" 13 "github.com/qri-io/dataset/stepfile" 14 "github.com/qri-io/qfs" 15 "github.com/qri-io/qri/base" 16 "github.com/qri-io/qri/repo" 17 repoTest "github.com/qri-io/qri/repo/test" 18 "github.com/qri-io/starlib" 19 "github.com/qri-io/starlib/testdata" 20 "go.starlark.net/starlark" 21 "go.starlark.net/starlarktest" 22 ) 23 24 func scriptFile(t *testing.T, path string) qfs.File { 25 data, err := ioutil.ReadFile(path) 26 if err != nil { 27 t.Fatal(err) 28 } 29 30 return qfs.NewMemfileBytes(path, data) 31 } 32 33 func TestOpts(t *testing.T) { 34 o := &ExecOpts{} 35 SetSecrets(nil)(o) 36 SetSecrets(map[string]string{"a": "b"})(o) 37 AddQriRepo(nil)(o) 38 39 expect := &ExecOpts{ 40 Secrets: map[string]interface{}{"a": "b"}, 41 ErrWriter: nil, 42 } 43 44 if diff := cmp.Diff(expect, o); diff != "" { 45 t.Errorf("result mismatch (-want +got):\n%s", diff) 46 } 47 } 48 49 func TestExecScript(t *testing.T) { 50 ctx := context.Background() 51 ds := &dataset.Dataset{ 52 Transform: &dataset.Transform{}, 53 } 54 ds.Transform.SetScriptFile(scriptFile(t, "testdata/tf.star")) 55 56 if err := ExecScript(ctx, ds); err != nil { 57 t.Fatal(err) 58 } 59 if ds.Transform == nil { 60 t.Error("expected transform") 61 } 62 63 entryReader, err := dsio.NewEntryReader(ds.Structure, ds.BodyFile()) 64 if err != nil { 65 t.Fatalf("couldn't create entry reader from returned dataset & body file: %s", err.Error()) 66 } 67 68 i := 0 69 dsio.EachEntry(entryReader, func(n int, x dsio.Entry, e error) error { 70 if e != nil { 71 t.Errorf("entry %d iteration error: %s", i, e.Error()) 72 } 73 i++ 74 return nil 75 }) 76 77 if i != 3 { 78 t.Errorf("expected 3 entries, got: %d", i) 79 } 80 if ds.Structure.Entries != 3 { 81 t.Errorf("expected `ds.Structure.Entries` to be 3, got: %d", i) 82 } 83 } 84 85 func TestExecScript2(t *testing.T) { 86 ctx := context.Background() 87 s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 88 w.Write([]byte(`{"foo":[["bar","baz","bat"]]}`)) 89 })) 90 91 ds := &dataset.Dataset{ 92 Transform: &dataset.Transform{}, 93 } 94 ds.Transform.SetScriptFile(scriptFile(t, "testdata/fetch.star")) 95 err := ExecScript(ctx, ds, func(o *ExecOpts) { 96 o.Globals["test_server_url"] = starlark.String(s.URL) 97 }) 98 99 if err != nil { 100 t.Error(err.Error()) 101 return 102 } 103 if ds.Transform == nil { 104 t.Error("expected transform") 105 } 106 } 107 108 func TestExecStep(t *testing.T) { 109 ctx := context.Background() 110 ds := &dataset.Dataset{ 111 Transform: &dataset.Transform{ 112 Steps: []*dataset.TransformStep{ 113 { 114 Name: "transform", 115 Syntax: "starlark", 116 Category: "transform", 117 Script: "ds = dataset.latest()\nds.body = [[1,2,3]]\ndataset.commit(ds)", 118 }, 119 }, 120 }, 121 } 122 // Run the single step. 123 stepRunner := NewStepRunner(ds) 124 err := stepRunner.RunStep(ctx, ds, ds.Transform.Steps[0]) 125 if err != nil { 126 t.Fatal(err) 127 } 128 // Check that body was set by the transform step. 129 bodyfile := ds.BodyFile() 130 if bodyfile == nil { 131 t.Fatal("dataset did not have body assigned") 132 } 133 // Check that the `ds.Structure.Entries` was set 134 if ds.Structure == nil { 135 t.Fatal("dataset does not have a structure assigned") 136 } 137 if ds.Structure.Entries != 1 { 138 t.Errorf("`ds.Structure.Entries` not assigned correctly, expected 1, got: %d", ds.Structure.Entries) 139 } 140 data, err := ioutil.ReadAll(bodyfile) 141 if err != nil { 142 t.Fatal(err) 143 } 144 actual := string(data) 145 expect := "1,2,3\n" 146 if actual != expect { 147 t.Errorf("expected: %q, actual: %q", expect, actual) 148 } 149 } 150 151 func TestEditMeta(t *testing.T) { 152 ctx := context.Background() 153 r := testRepo(t) 154 ds := &dataset.Dataset{ 155 Peername: "peer", 156 Name: "movies", 157 Transform: &dataset.Transform{}, 158 } 159 ds.Transform.SetScriptFile(scriptFile(t, "testdata/set_meta.star")) 160 161 err := ExecScript(ctx, ds, func(o *ExecOpts) { 162 o.ModuleLoader = testModuleLoader(t) 163 o.DatasetLoader = base.NewTestDatasetLoader(r.Filesystem(), r) 164 }) 165 if err != nil { 166 t.Fatal(err) 167 } 168 if ds.Meta.Title != "new title" { 169 t.Errorf("meta title was not changed") 170 } 171 if ds.Structure.Entries != 2335 { 172 t.Errorf("`ds.Structure.Entries` incorrect, expected 2335, got: %d", ds.Structure.Entries) 173 } 174 } 175 176 func TestScriptError(t *testing.T) { 177 ctx := context.Background() 178 script := `error("script error")` 179 scriptFile := qfs.NewMemfileBytes("tf.star", []byte(script)) 180 181 ds := &dataset.Dataset{ 182 Transform: &dataset.Transform{}, 183 } 184 ds.Transform.SetScriptFile(scriptFile) 185 if err := ExecScript(ctx, ds); err == nil { 186 t.Errorf("expected script to error. got nil") 187 } 188 } 189 190 func TestLoadDataset(t *testing.T) { 191 ctx := context.Background() 192 r := testRepo(t) 193 194 ds := &dataset.Dataset{ 195 Transform: &dataset.Transform{}, 196 } 197 ds.Transform.SetScriptFile(scriptFile(t, "testdata/load_ds.star")) 198 199 err := ExecScript(ctx, ds, func(o *ExecOpts) { 200 o.Repo = r 201 o.ModuleLoader = testModuleLoader(t) 202 o.DatasetLoader = base.NewTestDatasetLoader(r.Filesystem(), r) 203 }) 204 if err != nil { 205 t.Fatal(err) 206 } 207 } 208 209 func TestGetMetaNilPrev(t *testing.T) { 210 ctx := context.Background() 211 ds := &dataset.Dataset{ 212 Transform: &dataset.Transform{}, 213 } 214 ds.Transform.SetScriptFile(scriptFile(t, "testdata/meta_title.star")) 215 err := ExecScript(ctx, ds) 216 if err != nil { 217 t.Error(err.Error()) 218 return 219 } 220 bodyfile := ds.BodyFile() 221 if bodyfile == nil { 222 t.Fatal("dataset did not have body assigned") 223 } 224 data, _ := ioutil.ReadAll(bodyfile) 225 actual := string(data) 226 expect := "no title\n" 227 if actual != expect { 228 t.Errorf("expected: %q, actual: %q", expect, actual) 229 } 230 } 231 232 func TestGetMetaWithPrev(t *testing.T) { 233 ctx := context.Background() 234 ds := &dataset.Dataset{ 235 Meta: &dataset.Meta{ 236 Title: "test_title", 237 }, 238 Transform: &dataset.Transform{}, 239 } 240 ds.Transform.SetScriptFile(scriptFile(t, "testdata/meta_title.star")) 241 err := ExecScript(ctx, ds) 242 if err != nil { 243 t.Error(err.Error()) 244 return 245 } 246 bodyfile := ds.BodyFile() 247 if bodyfile == nil { 248 t.Fatal("dataset did not have body assigned") 249 } 250 data, _ := ioutil.ReadAll(bodyfile) 251 actual := string(data) 252 expect := "title: test_title\n" 253 if actual != expect { 254 t.Errorf("expected: %q, actual: %q", expect, actual) 255 } 256 } 257 258 func testRepo(t *testing.T) repo.Repo { 259 mr, err := repoTest.NewTestRepo() 260 if err != nil { 261 t.Fatal(err) 262 } 263 return mr 264 } 265 266 func testModuleLoader(t *testing.T) func(thread *starlark.Thread, module string) (dict starlark.StringDict, err error) { 267 assertLoader := testdata.NewLoader(nil, "") 268 return func(thread *starlark.Thread, module string) (dict starlark.StringDict, err error) { 269 starlarktest.SetReporter(thread, t) 270 if module == "assert.star" { 271 return assertLoader(thread, module) 272 } 273 return starlib.Loader(thread, module) 274 } 275 } 276 277 func ExecScript(ctx context.Context, ds *dataset.Dataset, opts ...func(o *ExecOpts)) error { 278 // Convert single-file transform scripts to steps 279 if len(ds.Transform.Steps) == 0 && ds.Transform.ScriptFile() != nil { 280 steps, err := stepfile.Read(ds.Transform.ScriptFile()) 281 if err != nil { 282 return err 283 } 284 ds.Transform.Steps = steps 285 } 286 287 runner := NewStepRunner(ds, opts...) 288 for _, step := range ds.Transform.Steps { 289 if err := runner.RunStep(ctx, ds, step); err != nil { 290 return err 291 } 292 } 293 return nil 294 }