github.com/raarceoml/godog@v0.7.9/suite.go (about) 1 package godog 2 3 import ( 4 "bytes" 5 "fmt" 6 "io" 7 "math/rand" 8 "os" 9 "path/filepath" 10 "reflect" 11 "regexp" 12 "sort" 13 "strconv" 14 "strings" 15 "unicode/utf8" 16 17 "github.com/DATA-DOG/godog/gherkin" 18 ) 19 20 var errorInterface = reflect.TypeOf((*error)(nil)).Elem() 21 var typeOfBytes = reflect.TypeOf([]byte(nil)) 22 23 type feature struct { 24 *gherkin.Feature 25 Content []byte `json:"-"` 26 Path string `json:"path"` 27 scenarios map[int]bool 28 order int 29 } 30 31 // ErrUndefined is returned in case if step definition was not found 32 var ErrUndefined = fmt.Errorf("step is undefined") 33 34 // ErrPending should be returned by step definition if 35 // step implementation is pending 36 var ErrPending = fmt.Errorf("step implementation is pending") 37 38 // Suite allows various contexts 39 // to register steps and event handlers. 40 // 41 // When running a test suite, the instance of Suite 42 // is passed to all functions (contexts), which 43 // have it as a first and only argument. 44 // 45 // Note that all event hooks does not catch panic errors 46 // in order to have a trace information. Only step 47 // executions are catching panic error since it may 48 // be a context specific error. 49 type Suite struct { 50 steps []*StepDef 51 features []*feature 52 fmt Formatter 53 54 failed bool 55 randomSeed int64 56 stopOnFailure bool 57 strict bool 58 59 // suite event handlers 60 beforeSuiteHandlers []func() 61 beforeFeatureHandlers []func(*gherkin.Feature) 62 beforeScenarioHandlers []func(interface{}) 63 beforeStepHandlers []func(*gherkin.Step) 64 afterStepHandlers []func(*gherkin.Step, error) 65 afterScenarioHandlers []func(interface{}, error) 66 afterFeatureHandlers []func(*gherkin.Feature) 67 afterSuiteHandlers []func() 68 } 69 70 // Step allows to register a *StepDef in Godog 71 // feature suite, the definition will be applied 72 // to all steps matching the given Regexp expr. 73 // 74 // It will panic if expr is not a valid regular 75 // expression or stepFunc is not a valid step 76 // handler. 77 // 78 // Note that if there are two definitions which may match 79 // the same step, then only the first matched handler 80 // will be applied. 81 // 82 // If none of the *StepDef is matched, then 83 // ErrUndefined error will be returned when 84 // running steps. 85 func (s *Suite) Step(expr interface{}, stepFunc interface{}) { 86 var regex *regexp.Regexp 87 88 switch t := expr.(type) { 89 case *regexp.Regexp: 90 regex = t 91 case string: 92 regex = regexp.MustCompile(t) 93 case []byte: 94 regex = regexp.MustCompile(string(t)) 95 default: 96 panic(fmt.Sprintf("expecting expr to be a *regexp.Regexp or a string, got type: %T", expr)) 97 } 98 99 v := reflect.ValueOf(stepFunc) 100 typ := v.Type() 101 if typ.Kind() != reflect.Func { 102 panic(fmt.Sprintf("expected handler to be func, but got: %T", stepFunc)) 103 } 104 105 if typ.NumOut() != 1 { 106 panic(fmt.Sprintf("expected handler to return only one value, but it has: %d", typ.NumOut())) 107 } 108 109 def := &StepDef{ 110 Handler: stepFunc, 111 Expr: regex, 112 hv: v, 113 } 114 115 typ = typ.Out(0) 116 switch typ.Kind() { 117 case reflect.Interface: 118 if !typ.Implements(errorInterface) { 119 panic(fmt.Sprintf("expected handler to return an error, but got: %s", typ.Kind())) 120 } 121 case reflect.Slice: 122 if typ.Elem().Kind() != reflect.String { 123 panic(fmt.Sprintf("expected handler to return []string for multistep, but got: []%s", typ.Kind())) 124 } 125 def.nested = true 126 default: 127 panic(fmt.Sprintf("expected handler to return an error or []string, but got: %s", typ.Kind())) 128 } 129 130 s.steps = append(s.steps, def) 131 } 132 133 // BeforeSuite registers a function or method 134 // to be run once before suite runner. 135 // 136 // Use it to prepare the test suite for a spin. 137 // Connect and prepare database for instance... 138 func (s *Suite) BeforeSuite(fn func()) { 139 s.beforeSuiteHandlers = append(s.beforeSuiteHandlers, fn) 140 } 141 142 // BeforeFeature registers a function or method 143 // to be run once before every feature execution. 144 // 145 // If godog is run with concurrency option, it will 146 // run every feature per goroutine. So user may choose 147 // whether to isolate state within feature context or 148 // scenario. 149 // 150 // Best practice is not to have any state dependency on 151 // every scenario, but in some cases if VM for example 152 // needs to be started it may take very long for each 153 // scenario to restart it. 154 // 155 // Use it wisely and avoid sharing state between scenarios. 156 func (s *Suite) BeforeFeature(fn func(*gherkin.Feature)) { 157 s.beforeFeatureHandlers = append(s.beforeFeatureHandlers, fn) 158 } 159 160 // BeforeScenario registers a function or method 161 // to be run before every scenario or scenario outline. 162 // 163 // The interface argument may be *gherkin.Scenario 164 // or *gherkin.ScenarioOutline 165 // 166 // It is a good practice to restore the default state 167 // before every scenario so it would be isolated from 168 // any kind of state. 169 func (s *Suite) BeforeScenario(fn func(interface{})) { 170 s.beforeScenarioHandlers = append(s.beforeScenarioHandlers, fn) 171 } 172 173 // BeforeStep registers a function or method 174 // to be run before every scenario 175 func (s *Suite) BeforeStep(fn func(*gherkin.Step)) { 176 s.beforeStepHandlers = append(s.beforeStepHandlers, fn) 177 } 178 179 // AfterStep registers an function or method 180 // to be run after every scenario 181 // 182 // It may be convenient to return a different kind of error 183 // in order to print more state details which may help 184 // in case of step failure 185 // 186 // In some cases, for example when running a headless 187 // browser, to take a screenshot after failure. 188 func (s *Suite) AfterStep(fn func(*gherkin.Step, error)) { 189 s.afterStepHandlers = append(s.afterStepHandlers, fn) 190 } 191 192 // AfterScenario registers an function or method 193 // to be run after every scenario or scenario outline 194 // 195 // The interface argument may be *gherkin.Scenario 196 // or *gherkin.ScenarioOutline 197 func (s *Suite) AfterScenario(fn func(interface{}, error)) { 198 s.afterScenarioHandlers = append(s.afterScenarioHandlers, fn) 199 } 200 201 // AfterFeature registers a function or method 202 // to be run once after feature executed all scenarios. 203 func (s *Suite) AfterFeature(fn func(*gherkin.Feature)) { 204 s.afterFeatureHandlers = append(s.afterFeatureHandlers, fn) 205 } 206 207 // AfterSuite registers a function or method 208 // to be run once after suite runner 209 func (s *Suite) AfterSuite(fn func()) { 210 s.afterSuiteHandlers = append(s.afterSuiteHandlers, fn) 211 } 212 213 func (s *Suite) run() { 214 // run before suite handlers 215 for _, f := range s.beforeSuiteHandlers { 216 f() 217 } 218 // run features 219 for _, f := range s.features { 220 s.runFeature(f) 221 if s.failed && s.stopOnFailure { 222 // stop on first failure 223 break 224 } 225 } 226 // run after suite handlers 227 for _, f := range s.afterSuiteHandlers { 228 f() 229 } 230 } 231 232 func (s *Suite) matchStep(step *gherkin.Step) *StepDef { 233 def := s.matchStepText(step.Text) 234 if def != nil && step.Argument != nil { 235 def.args = append(def.args, step.Argument) 236 } 237 return def 238 } 239 240 func (s *Suite) runStep(step *gherkin.Step, prevStepErr error) (err error) { 241 // run before step handlers 242 for _, f := range s.beforeStepHandlers { 243 f(step) 244 } 245 246 match := s.matchStep(step) 247 s.fmt.Defined(step, match) 248 249 // user multistep definitions may panic 250 defer func() { 251 if e := recover(); e != nil { 252 err = &traceError{ 253 msg: fmt.Sprintf("%v", e), 254 stack: callStack(), 255 } 256 } 257 258 if prevStepErr != nil { 259 return 260 } 261 262 if err == ErrUndefined { 263 return 264 } 265 266 switch err { 267 case nil: 268 s.fmt.Passed(step, match) 269 case ErrPending: 270 s.fmt.Pending(step, match) 271 default: 272 s.fmt.Failed(step, match, err) 273 } 274 275 // run after step handlers 276 for _, f := range s.afterStepHandlers { 277 f(step, err) 278 } 279 }() 280 281 if undef, err := s.maybeUndefined(step.Text, step.Argument); err != nil { 282 return err 283 } else if len(undef) > 0 { 284 if match != nil { 285 match = &StepDef{ 286 args: match.args, 287 hv: match.hv, 288 Expr: match.Expr, 289 Handler: match.Handler, 290 nested: match.nested, 291 undefined: undef, 292 } 293 } 294 s.fmt.Undefined(step, match) 295 return ErrUndefined 296 } 297 298 if prevStepErr != nil { 299 s.fmt.Skipped(step, match) 300 return nil 301 } 302 303 err = s.maybeSubSteps(match.run()) 304 return 305 } 306 307 func (s *Suite) maybeUndefined(text string, arg interface{}) ([]string, error) { 308 step := s.matchStepText(text) 309 if nil == step { 310 return []string{text}, nil 311 } 312 313 var undefined []string 314 if !step.nested { 315 return undefined, nil 316 } 317 318 if arg != nil { 319 step.args = append(step.args, arg) 320 } 321 322 for _, next := range step.run().(Steps) { 323 lines := strings.Split(next, "\n") 324 // @TODO: we cannot currently parse table or content body from nested steps 325 if len(lines) > 1 { 326 return undefined, fmt.Errorf("nested steps cannot be multiline and have table or content body argument") 327 } 328 if len(lines[0]) > 0 && lines[0][len(lines[0])-1] == ':' { 329 return undefined, fmt.Errorf("nested steps cannot be multiline and have table or content body argument") 330 } 331 undef, err := s.maybeUndefined(next, nil) 332 if err != nil { 333 return undefined, err 334 } 335 undefined = append(undefined, undef...) 336 } 337 return undefined, nil 338 } 339 340 func (s *Suite) maybeSubSteps(result interface{}) error { 341 if nil == result { 342 return nil 343 } 344 345 if err, ok := result.(error); ok { 346 return err 347 } 348 349 steps, ok := result.(Steps) 350 if !ok { 351 return fmt.Errorf("unexpected error, should have been []string: %T - %+v", result, result) 352 } 353 354 for _, text := range steps { 355 if def := s.matchStepText(text); def == nil { 356 return ErrUndefined 357 } else if err := s.maybeSubSteps(def.run()); err != nil { 358 return fmt.Errorf("%s: %+v", text, err) 359 } 360 } 361 return nil 362 } 363 364 func (s *Suite) matchStepText(text string) *StepDef { 365 for _, h := range s.steps { 366 if m := h.Expr.FindStringSubmatch(text); len(m) > 0 { 367 var args []interface{} 368 for _, m := range m[1:] { 369 args = append(args, m) 370 } 371 372 // since we need to assign arguments 373 // better to copy the step definition 374 return &StepDef{ 375 args: args, 376 hv: h.hv, 377 Expr: h.Expr, 378 Handler: h.Handler, 379 nested: h.nested, 380 } 381 } 382 } 383 return nil 384 } 385 386 func (s *Suite) runSteps(steps []*gherkin.Step) (err error) { 387 for _, step := range steps { 388 stepErr := s.runStep(step, err) 389 switch stepErr { 390 case ErrUndefined: 391 // do not overwrite failed error 392 if err == ErrUndefined || err == nil { 393 err = stepErr 394 } 395 case ErrPending: 396 err = stepErr 397 case nil: 398 default: 399 err = stepErr 400 } 401 } 402 return 403 } 404 405 func (s *Suite) runOutline(outline *gherkin.ScenarioOutline, b *gherkin.Background) (failErr error) { 406 s.fmt.Node(outline) 407 408 for _, ex := range outline.Examples { 409 example, hasExamples := examples(ex) 410 if !hasExamples { 411 // @TODO: may need to print empty example node, but 412 // for backward compatibility, cannot cast to *gherkin.ExamplesBase 413 // at the moment 414 continue 415 } 416 417 s.fmt.Node(example) 418 placeholders := example.TableHeader.Cells 419 groups := example.TableBody 420 421 for _, group := range groups { 422 if !isEmptyScenario(outline) { 423 for _, f := range s.beforeScenarioHandlers { 424 f(outline) 425 } 426 } 427 var steps []*gherkin.Step 428 for _, outlineStep := range outline.Steps { 429 text := outlineStep.Text 430 for i, placeholder := range placeholders { 431 text = strings.Replace(text, "<"+placeholder.Value+">", group.Cells[i].Value, -1) 432 } 433 434 // translate argument 435 arg := outlineStep.Argument 436 switch t := outlineStep.Argument.(type) { 437 case *gherkin.DataTable: 438 tbl := &gherkin.DataTable{ 439 Node: t.Node, 440 Rows: make([]*gherkin.TableRow, len(t.Rows)), 441 } 442 for i, row := range t.Rows { 443 cells := make([]*gherkin.TableCell, len(row.Cells)) 444 for j, cell := range row.Cells { 445 trans := cell.Value 446 for i, placeholder := range placeholders { 447 trans = strings.Replace(trans, "<"+placeholder.Value+">", group.Cells[i].Value, -1) 448 } 449 cells[j] = &gherkin.TableCell{ 450 Node: cell.Node, 451 Value: trans, 452 } 453 } 454 tbl.Rows[i] = &gherkin.TableRow{ 455 Node: row.Node, 456 Cells: cells, 457 } 458 } 459 arg = tbl 460 case *gherkin.DocString: 461 trans := t.Content 462 for i, placeholder := range placeholders { 463 trans = strings.Replace(trans, "<"+placeholder.Value+">", group.Cells[i].Value, -1) 464 } 465 arg = &gherkin.DocString{ 466 Node: t.Node, 467 Content: trans, 468 ContentType: t.ContentType, 469 Delimitter: t.Delimitter, 470 } 471 } 472 473 // clone a step 474 step := &gherkin.Step{ 475 Node: outlineStep.Node, 476 Text: text, 477 Keyword: outlineStep.Keyword, 478 Argument: arg, 479 } 480 steps = append(steps, step) 481 } 482 483 // run example table row 484 s.fmt.Node(group) 485 486 if b != nil { 487 steps = append(b.Steps, steps...) 488 } 489 490 err := s.runSteps(steps) 491 492 if !isEmptyScenario(outline) { 493 for _, f := range s.afterScenarioHandlers { 494 f(outline, err) 495 } 496 } 497 498 if s.shouldFail(err) { 499 failErr = err 500 if s.stopOnFailure { 501 return 502 } 503 } 504 } 505 } 506 return 507 } 508 509 func (s *Suite) shouldFail(err error) bool { 510 if err == nil { 511 return false 512 } 513 514 if err == ErrUndefined || err == ErrPending { 515 return s.strict 516 } 517 518 return true 519 } 520 521 func (s *Suite) runFeature(f *feature) { 522 if !isEmptyFeature(f.Feature) { 523 for _, fn := range s.beforeFeatureHandlers { 524 fn(f.Feature) 525 } 526 } 527 528 s.fmt.Feature(f.Feature, f.Path, f.Content) 529 530 // make a local copy of the feature scenario defenitions, 531 // then shuffle it if we are randomizing scenarios 532 scenarios := make([]interface{}, len(f.ScenarioDefinitions)) 533 if s.randomSeed != 0 { 534 r := rand.New(rand.NewSource(s.randomSeed)) 535 perm := r.Perm(len(f.ScenarioDefinitions)) 536 for i, v := range perm { 537 scenarios[v] = f.ScenarioDefinitions[i] 538 } 539 } else { 540 copy(scenarios, f.ScenarioDefinitions) 541 } 542 543 defer func() { 544 if !isEmptyFeature(f.Feature) { 545 for _, fn := range s.afterFeatureHandlers { 546 fn(f.Feature) 547 } 548 } 549 }() 550 551 for _, scenario := range scenarios { 552 var err error 553 if f.Background != nil { 554 s.fmt.Node(f.Background) 555 } 556 switch t := scenario.(type) { 557 case *gherkin.ScenarioOutline: 558 err = s.runOutline(t, f.Background) 559 case *gherkin.Scenario: 560 err = s.runScenario(t, f.Background) 561 } 562 if s.shouldFail(err) { 563 s.failed = true 564 if s.stopOnFailure { 565 return 566 } 567 } 568 } 569 } 570 571 func (s *Suite) runScenario(scenario *gherkin.Scenario, b *gherkin.Background) (err error) { 572 if isEmptyScenario(scenario) { 573 s.fmt.Node(scenario) 574 return ErrUndefined 575 } 576 577 // run before scenario handlers 578 for _, f := range s.beforeScenarioHandlers { 579 f(scenario) 580 } 581 582 s.fmt.Node(scenario) 583 584 // background 585 steps := scenario.Steps 586 if b != nil { 587 steps = append(b.Steps, steps...) 588 } 589 590 // scenario 591 err = s.runSteps(steps) 592 593 // run after scenario handlers 594 for _, f := range s.afterScenarioHandlers { 595 f(scenario, err) 596 } 597 598 return 599 } 600 601 func (s *Suite) printStepDefinitions(w io.Writer) { 602 var longest int 603 for _, def := range s.steps { 604 n := utf8.RuneCountInString(def.Expr.String()) 605 if longest < n { 606 longest = n 607 } 608 } 609 for _, def := range s.steps { 610 n := utf8.RuneCountInString(def.Expr.String()) 611 location := def.definitionID() 612 spaces := strings.Repeat(" ", longest-n) 613 fmt.Fprintln(w, yellow(def.Expr.String())+spaces, black("# "+location)) 614 } 615 if len(s.steps) == 0 { 616 fmt.Fprintln(w, "there were no contexts registered, could not find any step definition..") 617 } 618 } 619 620 func parseFeatures(filter string, paths []string) ([]*feature, error) { 621 byPath := make(map[string]*feature) 622 var order int 623 for _, pat := range paths { 624 // check if line number is specified 625 parts := strings.Split(pat, ":") 626 path := parts[0] 627 line := -1 628 var err error 629 if len(parts) > 1 { 630 line, err = strconv.Atoi(parts[1]) 631 if err != nil { 632 return nil, fmt.Errorf("line number should follow after colon path delimiter") 633 } 634 } 635 // parse features 636 err = filepath.Walk(path, func(p string, f os.FileInfo, err error) error { 637 if err == nil && !f.IsDir() && strings.HasSuffix(p, ".feature") { 638 reader, err := os.Open(p) 639 if err != nil { 640 return err 641 } 642 var buf bytes.Buffer 643 ft, err := gherkin.ParseFeature(io.TeeReader(reader, &buf)) 644 reader.Close() 645 if err != nil { 646 return fmt.Errorf("%s - %v", p, err) 647 } 648 649 feat := byPath[p] 650 if feat == nil { 651 feat = &feature{ 652 Path: p, 653 Feature: ft, 654 Content: buf.Bytes(), 655 scenarios: make(map[int]bool), 656 order: order, 657 } 658 order++ 659 byPath[p] = feat 660 } 661 // filter scenario by line number 662 for _, def := range ft.ScenarioDefinitions { 663 var ln int 664 switch t := def.(type) { 665 case *gherkin.Scenario: 666 ln = t.Location.Line 667 case *gherkin.ScenarioOutline: 668 ln = t.Location.Line 669 } 670 if line == -1 || ln == line { 671 feat.scenarios[ln] = true 672 } 673 } 674 } 675 return err 676 }) 677 // check error 678 switch { 679 case os.IsNotExist(err): 680 return nil, fmt.Errorf(`feature path "%s" is not available`, path) 681 case os.IsPermission(err): 682 return nil, fmt.Errorf(`feature path "%s" is not accessible`, path) 683 case err != nil: 684 return nil, err 685 } 686 } 687 return filterFeatures(filter, byPath), nil 688 } 689 690 type sortByOrderGiven []*feature 691 692 func (s sortByOrderGiven) Len() int { return len(s) } 693 func (s sortByOrderGiven) Less(i, j int) bool { return s[i].order < s[j].order } 694 func (s sortByOrderGiven) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 695 696 func filterFeatures(tags string, collected map[string]*feature) (features []*feature) { 697 for _, ft := range collected { 698 var scenarios []interface{} 699 for _, def := range ft.ScenarioDefinitions { 700 var ln int 701 switch t := def.(type) { 702 case *gherkin.Scenario: 703 ln = t.Location.Line 704 case *gherkin.ScenarioOutline: 705 ln = t.Location.Line 706 } 707 if ft.scenarios[ln] { 708 scenarios = append(scenarios, def) 709 } 710 } 711 ft.ScenarioDefinitions = scenarios 712 applyTagFilter(tags, ft.Feature) 713 features = append(features, ft) 714 } 715 716 sort.Sort(sortByOrderGiven(features)) 717 718 return features 719 } 720 721 func applyTagFilter(tags string, ft *gherkin.Feature) { 722 if len(tags) == 0 { 723 return 724 } 725 726 var scenarios []interface{} 727 for _, scenario := range ft.ScenarioDefinitions { 728 switch t := scenario.(type) { 729 case *gherkin.ScenarioOutline: 730 var allExamples []*gherkin.Examples 731 for _, examples := range t.Examples { 732 if matchesTags(tags, allTags(ft, t, examples)) { 733 allExamples = append(allExamples, examples) 734 } 735 } 736 t.Examples = allExamples 737 if len(t.Examples) > 0 { 738 scenarios = append(scenarios, scenario) 739 } 740 case *gherkin.Scenario: 741 if matchesTags(tags, allTags(ft, t)) { 742 scenarios = append(scenarios, scenario) 743 } 744 } 745 } 746 ft.ScenarioDefinitions = scenarios 747 } 748 749 func allTags(nodes ...interface{}) []string { 750 var tags, tmp []string 751 for _, node := range nodes { 752 var gr []*gherkin.Tag 753 switch t := node.(type) { 754 case *gherkin.Feature: 755 gr = t.Tags 756 case *gherkin.ScenarioOutline: 757 gr = t.Tags 758 case *gherkin.Scenario: 759 gr = t.Tags 760 case *gherkin.Examples: 761 gr = t.Tags 762 } 763 764 for _, gtag := range gr { 765 tag := strings.TrimSpace(gtag.Name) 766 if tag[0] == '@' { 767 tag = tag[1:] 768 } 769 copy(tmp, tags) 770 var found bool 771 for _, tg := range tmp { 772 if tg == tag { 773 found = true 774 break 775 } 776 } 777 if !found { 778 tags = append(tags, tag) 779 } 780 } 781 } 782 return tags 783 } 784 785 func hasTag(tags []string, tag string) bool { 786 for _, t := range tags { 787 if t == tag { 788 return true 789 } 790 } 791 return false 792 } 793 794 // based on http://behat.readthedocs.org/en/v2.5/guides/6.cli.html#gherkin-filters 795 func matchesTags(filter string, tags []string) (ok bool) { 796 ok = true 797 for _, andTags := range strings.Split(filter, "&&") { 798 var okComma bool 799 for _, tag := range strings.Split(andTags, ",") { 800 tag = strings.Replace(strings.TrimSpace(tag), "@", "", -1) 801 if tag[0] == '~' { 802 tag = tag[1:] 803 okComma = !hasTag(tags, tag) || okComma 804 } else { 805 okComma = hasTag(tags, tag) || okComma 806 } 807 } 808 ok = ok && okComma 809 } 810 return 811 }