github.com/maps90/godog@v0.7.5-0.20170923143419-0093943021d4/suite.go (about) 1 package godog 2 3 import ( 4 "bytes" 5 "fmt" 6 "io" 7 "math/rand" 8 "os" 9 "path/filepath" 10 "reflect" 11 "regexp" 12 "strconv" 13 "strings" 14 "unicode/utf8" 15 16 "github.com/DATA-DOG/godog/gherkin" 17 ) 18 19 var errorInterface = reflect.TypeOf((*error)(nil)).Elem() 20 var typeOfBytes = reflect.TypeOf([]byte(nil)) 21 22 type feature struct { 23 *gherkin.Feature 24 Content []byte `json:"-"` 25 Path string `json:"path"` 26 } 27 28 // ErrUndefined is returned in case if step definition was not found 29 var ErrUndefined = fmt.Errorf("step is undefined") 30 31 // ErrPending should be returned by step definition if 32 // step implementation is pending 33 var ErrPending = fmt.Errorf("step implementation is pending") 34 35 // Suite allows various contexts 36 // to register steps and event handlers. 37 // 38 // When running a test suite, the instance of Suite 39 // is passed to all functions (contexts), which 40 // have it as a first and only argument. 41 // 42 // Note that all event hooks does not catch panic errors 43 // in order to have a trace information. Only step 44 // executions are catching panic error since it may 45 // be a context specific error. 46 type Suite struct { 47 steps []*StepDef 48 features []*feature 49 fmt Formatter 50 51 failed bool 52 randomSeed int64 53 stopOnFailure bool 54 strict bool 55 56 // suite event handlers 57 beforeSuiteHandlers []func() 58 beforeFeatureHandlers []func(*gherkin.Feature) 59 beforeScenarioHandlers []func(interface{}) 60 beforeStepHandlers []func(*gherkin.Step) 61 afterStepHandlers []func(*gherkin.Step, error) 62 afterScenarioHandlers []func(interface{}, error) 63 afterFeatureHandlers []func(*gherkin.Feature) 64 afterSuiteHandlers []func() 65 } 66 67 // Step allows to register a *StepDef in Godog 68 // feature suite, the definition will be applied 69 // to all steps matching the given Regexp expr. 70 // 71 // It will panic if expr is not a valid regular 72 // expression or stepFunc is not a valid step 73 // handler. 74 // 75 // Note that if there are two definitions which may match 76 // the same step, then only the first matched handler 77 // will be applied. 78 // 79 // If none of the *StepDef is matched, then 80 // ErrUndefined error will be returned when 81 // running steps. 82 func (s *Suite) Step(expr interface{}, stepFunc interface{}) { 83 var regex *regexp.Regexp 84 85 switch t := expr.(type) { 86 case *regexp.Regexp: 87 regex = t 88 case string: 89 regex = regexp.MustCompile(t) 90 case []byte: 91 regex = regexp.MustCompile(string(t)) 92 default: 93 panic(fmt.Sprintf("expecting expr to be a *regexp.Regexp or a string, got type: %T", expr)) 94 } 95 96 v := reflect.ValueOf(stepFunc) 97 typ := v.Type() 98 if typ.Kind() != reflect.Func { 99 panic(fmt.Sprintf("expected handler to be func, but got: %T", stepFunc)) 100 } 101 102 if typ.NumOut() != 1 { 103 panic(fmt.Sprintf("expected handler to return only one value, but it has: %d", typ.NumOut())) 104 } 105 106 def := &StepDef{ 107 Handler: stepFunc, 108 Expr: regex, 109 hv: v, 110 } 111 112 typ = typ.Out(0) 113 switch typ.Kind() { 114 case reflect.Interface: 115 if !typ.Implements(errorInterface) { 116 panic(fmt.Sprintf("expected handler to return an error, but got: %s", typ.Kind())) 117 } 118 case reflect.Slice: 119 if typ.Elem().Kind() != reflect.String { 120 panic(fmt.Sprintf("expected handler to return []string for multistep, but got: []%s", typ.Kind())) 121 } 122 def.nested = true 123 default: 124 panic(fmt.Sprintf("expected handler to return an error or []string, but got: %s", typ.Kind())) 125 } 126 127 s.steps = append(s.steps, def) 128 } 129 130 // BeforeSuite registers a function or method 131 // to be run once before suite runner. 132 // 133 // Use it to prepare the test suite for a spin. 134 // Connect and prepare database for instance... 135 func (s *Suite) BeforeSuite(fn func()) { 136 s.beforeSuiteHandlers = append(s.beforeSuiteHandlers, fn) 137 } 138 139 // BeforeFeature registers a function or method 140 // to be run once before every feature execution. 141 // 142 // If godog is run with concurrency option, it will 143 // run every feature per goroutine. So user may choose 144 // whether to isolate state within feature context or 145 // scenario. 146 // 147 // Best practice is not to have any state dependency on 148 // every scenario, but in some cases if VM for example 149 // needs to be started it may take very long for each 150 // scenario to restart it. 151 // 152 // Use it wisely and avoid sharing state between scenarios. 153 func (s *Suite) BeforeFeature(fn func(*gherkin.Feature)) { 154 s.beforeFeatureHandlers = append(s.beforeFeatureHandlers, fn) 155 } 156 157 // BeforeScenario registers a function or method 158 // to be run before every scenario or scenario outline. 159 // 160 // The interface argument may be *gherkin.Scenario 161 // or *gherkin.ScenarioOutline 162 // 163 // It is a good practice to restore the default state 164 // before every scenario so it would be isolated from 165 // any kind of state. 166 func (s *Suite) BeforeScenario(fn func(interface{})) { 167 s.beforeScenarioHandlers = append(s.beforeScenarioHandlers, fn) 168 } 169 170 // BeforeStep registers a function or method 171 // to be run before every scenario 172 func (s *Suite) BeforeStep(fn func(*gherkin.Step)) { 173 s.beforeStepHandlers = append(s.beforeStepHandlers, fn) 174 } 175 176 // AfterStep registers an function or method 177 // to be run after every scenario 178 // 179 // It may be convenient to return a different kind of error 180 // in order to print more state details which may help 181 // in case of step failure 182 // 183 // In some cases, for example when running a headless 184 // browser, to take a screenshot after failure. 185 func (s *Suite) AfterStep(fn func(*gherkin.Step, error)) { 186 s.afterStepHandlers = append(s.afterStepHandlers, fn) 187 } 188 189 // AfterScenario registers an function or method 190 // to be run after every scenario or scenario outline 191 // 192 // The interface argument may be *gherkin.Scenario 193 // or *gherkin.ScenarioOutline 194 func (s *Suite) AfterScenario(fn func(interface{}, error)) { 195 s.afterScenarioHandlers = append(s.afterScenarioHandlers, fn) 196 } 197 198 // AfterFeature registers a function or method 199 // to be run once after feature executed all scenarios. 200 func (s *Suite) AfterFeature(fn func(*gherkin.Feature)) { 201 s.afterFeatureHandlers = append(s.afterFeatureHandlers, fn) 202 } 203 204 // AfterSuite registers a function or method 205 // to be run once after suite runner 206 func (s *Suite) AfterSuite(fn func()) { 207 s.afterSuiteHandlers = append(s.afterSuiteHandlers, fn) 208 } 209 210 func (s *Suite) run() { 211 // run before suite handlers 212 for _, f := range s.beforeSuiteHandlers { 213 f() 214 } 215 // run features 216 for _, f := range s.features { 217 s.runFeature(f) 218 if s.failed && s.stopOnFailure { 219 // stop on first failure 220 break 221 } 222 } 223 // run after suite handlers 224 for _, f := range s.afterSuiteHandlers { 225 f() 226 } 227 } 228 229 func (s *Suite) matchStep(step *gherkin.Step) *StepDef { 230 def := s.matchStepText(step.Text) 231 if def != nil && step.Argument != nil { 232 def.args = append(def.args, step.Argument) 233 } 234 return def 235 } 236 237 func (s *Suite) runStep(step *gherkin.Step, prevStepErr error) (err error) { 238 match := s.matchStep(step) 239 s.fmt.Defined(step, match) 240 241 // user multistep definitions may panic 242 defer func() { 243 if e := recover(); e != nil { 244 err = &traceError{ 245 msg: fmt.Sprintf("%v", e), 246 stack: callStack(), 247 } 248 } 249 250 if prevStepErr != nil { 251 return 252 } 253 254 if err == ErrUndefined { 255 return 256 } 257 258 switch err { 259 case nil: 260 s.fmt.Passed(step, match) 261 case ErrPending: 262 s.fmt.Pending(step, match) 263 default: 264 s.fmt.Failed(step, match, err) 265 } 266 267 // run after step handlers 268 for _, f := range s.afterStepHandlers { 269 f(step, err) 270 } 271 }() 272 273 if undef, err := s.maybeUndefined(step.Text, step.Argument); err != nil { 274 return err 275 } else if len(undef) > 0 { 276 if match != nil { 277 match = &StepDef{ 278 args: match.args, 279 hv: match.hv, 280 Expr: match.Expr, 281 Handler: match.Handler, 282 nested: match.nested, 283 undefined: undef, 284 } 285 } 286 s.fmt.Undefined(step, match) 287 return ErrUndefined 288 } 289 290 if prevStepErr != nil { 291 s.fmt.Skipped(step, match) 292 return nil 293 } 294 295 // run before step handlers 296 for _, f := range s.beforeStepHandlers { 297 f(step) 298 } 299 300 err = s.maybeSubSteps(match.run()) 301 return 302 } 303 304 func (s *Suite) maybeUndefined(text string, arg interface{}) ([]string, error) { 305 step := s.matchStepText(text) 306 if nil == step { 307 return []string{text}, nil 308 } 309 310 var undefined []string 311 if !step.nested { 312 return undefined, nil 313 } 314 315 if arg != nil { 316 step.args = append(step.args, arg) 317 } 318 319 for _, next := range step.run().(Steps) { 320 lines := strings.Split(next, "\n") 321 // @TODO: we cannot currently parse table or content body from nested steps 322 if len(lines) > 1 { 323 return undefined, fmt.Errorf("nested steps cannot be multiline and have table or content body argument") 324 } 325 if len(lines[0]) > 0 && lines[0][len(lines[0])-1] == ':' { 326 return undefined, fmt.Errorf("nested steps cannot be multiline and have table or content body argument") 327 } 328 undef, err := s.maybeUndefined(next, nil) 329 if err != nil { 330 return undefined, err 331 } 332 undefined = append(undefined, undef...) 333 } 334 return undefined, nil 335 } 336 337 func (s *Suite) maybeSubSteps(result interface{}) error { 338 if nil == result { 339 return nil 340 } 341 342 if err, ok := result.(error); ok { 343 return err 344 } 345 346 steps, ok := result.(Steps) 347 if !ok { 348 return fmt.Errorf("unexpected error, should have been []string: %T - %+v", result, result) 349 } 350 351 for _, text := range steps { 352 if def := s.matchStepText(text); def == nil { 353 return ErrUndefined 354 } else if err := s.maybeSubSteps(def.run()); err != nil { 355 return fmt.Errorf("%s: %+v", text, err) 356 } 357 } 358 return nil 359 } 360 361 func (s *Suite) matchStepText(text string) *StepDef { 362 for _, h := range s.steps { 363 if m := h.Expr.FindStringSubmatch(text); len(m) > 0 { 364 var args []interface{} 365 for _, m := range m[1:] { 366 args = append(args, m) 367 } 368 369 // since we need to assign arguments 370 // better to copy the step definition 371 return &StepDef{ 372 args: args, 373 hv: h.hv, 374 Expr: h.Expr, 375 Handler: h.Handler, 376 nested: h.nested, 377 } 378 } 379 } 380 return nil 381 } 382 383 func (s *Suite) runSteps(steps []*gherkin.Step) (err error) { 384 for _, step := range steps { 385 stepErr := s.runStep(step, err) 386 switch stepErr { 387 case ErrUndefined: 388 // do not overwrite failed error 389 if err == ErrUndefined || err == nil { 390 err = stepErr 391 } 392 case ErrPending: 393 err = stepErr 394 case nil: 395 default: 396 err = stepErr 397 } 398 } 399 return 400 } 401 402 func (s *Suite) skipSteps(steps []*gherkin.Step) { 403 for _, step := range steps { 404 s.fmt.Skipped(step, s.matchStep(step)) 405 } 406 } 407 408 func (s *Suite) runOutline(outline *gherkin.ScenarioOutline, b *gherkin.Background) (failErr error) { 409 s.fmt.Node(outline) 410 411 for _, ex := range outline.Examples { 412 example, hasExamples := examples(ex) 413 if !hasExamples { 414 // @TODO: may need to print empty example node, but 415 // for backward compatibility, cannot cast to *gherkin.ExamplesBase 416 // at the moment 417 continue 418 } 419 420 s.fmt.Node(example) 421 placeholders := example.TableHeader.Cells 422 groups := example.TableBody 423 424 for _, group := range groups { 425 for _, f := range s.beforeScenarioHandlers { 426 f(outline) 427 } 428 var steps []*gherkin.Step 429 for _, outlineStep := range outline.Steps { 430 text := outlineStep.Text 431 for i, placeholder := range placeholders { 432 text = strings.Replace(text, "<"+placeholder.Value+">", group.Cells[i].Value, -1) 433 } 434 435 // translate argument 436 arg := outlineStep.Argument 437 switch t := outlineStep.Argument.(type) { 438 case *gherkin.DataTable: 439 tbl := &gherkin.DataTable{ 440 Node: t.Node, 441 Rows: make([]*gherkin.TableRow, len(t.Rows)), 442 } 443 for i, row := range t.Rows { 444 cells := make([]*gherkin.TableCell, len(row.Cells)) 445 for j, cell := range row.Cells { 446 trans := cell.Value 447 for i, placeholder := range placeholders { 448 trans = strings.Replace(trans, "<"+placeholder.Value+">", group.Cells[i].Value, -1) 449 } 450 cells[j] = &gherkin.TableCell{ 451 Node: cell.Node, 452 Value: trans, 453 } 454 } 455 tbl.Rows[i] = &gherkin.TableRow{ 456 Node: row.Node, 457 Cells: cells, 458 } 459 } 460 arg = tbl 461 case *gherkin.DocString: 462 trans := t.Content 463 for i, placeholder := range placeholders { 464 trans = strings.Replace(trans, "<"+placeholder.Value+">", group.Cells[i].Value, -1) 465 } 466 arg = &gherkin.DocString{ 467 Node: t.Node, 468 Content: trans, 469 ContentType: t.ContentType, 470 Delimitter: t.Delimitter, 471 } 472 } 473 474 // clone a step 475 step := &gherkin.Step{ 476 Node: outlineStep.Node, 477 Text: text, 478 Keyword: outlineStep.Keyword, 479 Argument: arg, 480 } 481 steps = append(steps, step) 482 } 483 484 // run example table row 485 s.fmt.Node(group) 486 487 if b != nil { 488 steps = append(b.Steps, steps...) 489 } 490 491 err := s.runSteps(steps) 492 493 for _, f := range s.afterScenarioHandlers { 494 f(outline, err) 495 } 496 497 if s.shouldFail(err) { 498 failErr = err 499 if s.stopOnFailure { 500 return 501 } 502 } 503 } 504 } 505 return 506 } 507 508 func (s *Suite) shouldFail(err error) bool { 509 if err == nil { 510 return false 511 } 512 513 if err == ErrUndefined || err == ErrPending { 514 return s.strict 515 } 516 517 return true 518 } 519 520 func (s *Suite) runFeature(f *feature) { 521 for _, fn := range s.beforeFeatureHandlers { 522 fn(f.Feature) 523 } 524 525 s.fmt.Feature(f.Feature, f.Path, f.Content) 526 527 // make a local copy of the feature scenario defenitions, 528 // then shuffle it if we are randomizing scenarios 529 scenarios := make([]interface{}, len(f.ScenarioDefinitions)) 530 if s.randomSeed != 0 { 531 r := rand.New(rand.NewSource(s.randomSeed)) 532 perm := r.Perm(len(f.ScenarioDefinitions)) 533 for i, v := range perm { 534 scenarios[v] = f.ScenarioDefinitions[i] 535 } 536 } else { 537 copy(scenarios, f.ScenarioDefinitions) 538 } 539 540 defer func() { 541 for _, fn := range s.afterFeatureHandlers { 542 fn(f.Feature) 543 } 544 }() 545 546 for _, scenario := range scenarios { 547 var err error 548 if f.Background != nil { 549 s.fmt.Node(f.Background) 550 } 551 switch t := scenario.(type) { 552 case *gherkin.ScenarioOutline: 553 err = s.runOutline(t, f.Background) 554 case *gherkin.Scenario: 555 err = s.runScenario(t, f.Background) 556 } 557 if s.shouldFail(err) { 558 s.failed = true 559 if s.stopOnFailure { 560 return 561 } 562 } 563 } 564 } 565 566 func (s *Suite) runScenario(scenario *gherkin.Scenario, b *gherkin.Background) (err error) { 567 // run before scenario handlers 568 for _, f := range s.beforeScenarioHandlers { 569 f(scenario) 570 } 571 572 s.fmt.Node(scenario) 573 574 // background 575 steps := scenario.Steps 576 if b != nil { 577 steps = append(b.Steps, steps...) 578 } 579 580 // scenario 581 err = s.runSteps(steps) 582 583 // run after scenario handlers 584 for _, f := range s.afterScenarioHandlers { 585 f(scenario, err) 586 } 587 588 return 589 } 590 591 func (s *Suite) printStepDefinitions(w io.Writer) { 592 var longest int 593 for _, def := range s.steps { 594 n := utf8.RuneCountInString(def.Expr.String()) 595 if longest < n { 596 longest = n 597 } 598 } 599 for _, def := range s.steps { 600 n := utf8.RuneCountInString(def.Expr.String()) 601 location := def.definitionID() 602 spaces := strings.Repeat(" ", longest-n) 603 fmt.Fprintln(w, yellow(def.Expr.String())+spaces, black("# "+location)) 604 } 605 if len(s.steps) == 0 { 606 fmt.Fprintln(w, "there were no contexts registered, could not find any step definition..") 607 } 608 } 609 610 func parseFeatures(filter string, paths []string) (features []*feature, err error) { 611 for _, pat := range paths { 612 // check if line number is specified 613 parts := strings.Split(pat, ":") 614 path := parts[0] 615 line := -1 616 if len(parts) > 1 { 617 line, err = strconv.Atoi(parts[1]) 618 if err != nil { 619 return features, fmt.Errorf("line number should follow after colon path delimiter") 620 } 621 } 622 // parse features 623 err = filepath.Walk(path, func(p string, f os.FileInfo, err error) error { 624 if err == nil && !f.IsDir() && strings.HasSuffix(p, ".feature") { 625 reader, err := os.Open(p) 626 if err != nil { 627 return err 628 } 629 var buf bytes.Buffer 630 ft, err := gherkin.ParseFeature(io.TeeReader(reader, &buf)) 631 reader.Close() 632 if err != nil { 633 return fmt.Errorf("%s - %v", p, err) 634 } 635 features = append(features, &feature{Path: p, Feature: ft, Content: buf.Bytes()}) 636 // filter scenario by line number 637 if line != -1 { 638 var scenarios []interface{} 639 for _, def := range ft.ScenarioDefinitions { 640 var ln int 641 switch t := def.(type) { 642 case *gherkin.Scenario: 643 ln = t.Location.Line 644 case *gherkin.ScenarioOutline: 645 ln = t.Location.Line 646 } 647 if ln == line { 648 scenarios = append(scenarios, def) 649 break 650 } 651 } 652 ft.ScenarioDefinitions = scenarios 653 } 654 applyTagFilter(filter, ft) 655 } 656 return err 657 }) 658 // check error 659 switch { 660 case os.IsNotExist(err): 661 return features, fmt.Errorf(`feature path "%s" is not available`, path) 662 case os.IsPermission(err): 663 return features, fmt.Errorf(`feature path "%s" is not accessible`, path) 664 case err != nil: 665 return features, err 666 } 667 } 668 return 669 } 670 671 func applyTagFilter(tags string, ft *gherkin.Feature) { 672 if len(tags) == 0 { 673 return 674 } 675 676 var scenarios []interface{} 677 for _, scenario := range ft.ScenarioDefinitions { 678 if matchesTags(tags, allTags(ft, scenario)) { 679 scenarios = append(scenarios, scenario) 680 } 681 } 682 ft.ScenarioDefinitions = scenarios 683 } 684 685 func allTags(nodes ...interface{}) []string { 686 var tags, tmp []string 687 for _, node := range nodes { 688 var gr []*gherkin.Tag 689 switch t := node.(type) { 690 case *gherkin.Feature: 691 gr = t.Tags 692 case *gherkin.ScenarioOutline: 693 gr = t.Tags 694 case *gherkin.Scenario: 695 gr = t.Tags 696 case *gherkin.Examples: 697 gr = t.Tags 698 } 699 700 for _, gtag := range gr { 701 tag := strings.TrimSpace(gtag.Name) 702 if tag[0] == '@' { 703 tag = tag[1:] 704 } 705 copy(tmp, tags) 706 var found bool 707 for _, tg := range tmp { 708 if tg == tag { 709 found = true 710 break 711 } 712 } 713 if !found { 714 tags = append(tags, tag) 715 } 716 } 717 } 718 return tags 719 } 720 721 func hasTag(tags []string, tag string) bool { 722 for _, t := range tags { 723 if t == tag { 724 return true 725 } 726 } 727 return false 728 } 729 730 // based on http://behat.readthedocs.org/en/v2.5/guides/6.cli.html#gherkin-filters 731 func matchesTags(filter string, tags []string) (ok bool) { 732 ok = true 733 for _, andTags := range strings.Split(filter, "&&") { 734 var okComma bool 735 for _, tag := range strings.Split(andTags, ",") { 736 tag = strings.Replace(strings.TrimSpace(tag), "@", "", -1) 737 if tag[0] == '~' { 738 tag = tag[1:] 739 okComma = !hasTag(tags, tag) || okComma 740 } else { 741 okComma = hasTag(tags, tag) || okComma 742 } 743 } 744 ok = (false != okComma && ok && okComma) || false 745 } 746 return 747 }