src.elv.sh@v0.21.0-dev.0.20240515223629-06979efb9a2a/pkg/eval/evaltest/test_transcript.go

// Package evaltest supports testing the Elvish interpreter and libraries.
//
// The entrypoint of this package is [TestTranscriptsInFS]. Typical usage looks
// like this:
//
//	import (
//		"embed"
//		"src.elv.sh/pkg/eval/evaltest"
//	)
//
//	//go:embed *.elv *.elvts
//	var transcripts embed.FS
//
//	func TestTranscripts(t *testing.T) {
//		evaltest.TestTranscriptsInFS(t, transcripts)
//	}
//
// See [src.elv.sh/pkg/transcript] for how transcript sessions are discovered.
//
// # Setup functions
//
// [TestTranscriptsInFS] accepts variadic arguments in (name, f) pairs, where
// name must not contain any spaces. Each pair defines a setup function that may
// be referred to in the transcripts with the directive "//name".
//
// The setup function f may take a *testing.T, *eval.Evaler and a string
// argument. All of them are optional but must appear in that order. If it takes
// a string argument, the directive can be followed by an argument after a space
// ("//name argument"), and that argument is passed to f. The argument itself
// may contain spaces.
//
// The following setup functions are predefined:
//
//   - skip-test: Don't run this test. Useful for examples in .d.elv files that
//     shouldn't be run as tests.
//
//   - in-temp-dir: Run inside a temporary directory.
//
//   - set-env $name $value: Run with the environment variable $name set to
//     $value.
//
//   - unset-env $name: Run with the environment variable $name unset.
//
//   - eval $code: Evaluate the argument as Elvish code.
//
//   - only-on $cond: Evaluate $cond like a //go:build constraint and only
//     run the test if the constraint is satisfied.
//
//     The syntax is the same as //go:build constraints, but the set of
//     supported tags is different and consists of: GOARCH and GOOS values,
//     "unix", "32bit" and "64bit".
//
//   - deprecation-level $x: Run with deprecation level set to $x.
//
// These setup functions can then be used in transcripts as directives. By
// default, they only apply to the current session; adding a "each:" prefix
// makes them apply to descendant sessions too.
//
//	//global-setup
//	//each:global-setup-2
//
//	# h1 #
//	//h1-setup
//	//each:h1-setup2
//
//	## h2 ##
//	//h2-setup
//
//	// All of globa-setup2, h1-setup2 and h2-setup are run for this session, in
//	// that
//
//	~> echo foo
//	foo
//
// # ELVISH_TRANSCRIPT_RUN
//
// The environment variable ELVISH_TRANSCRIPT_RUN may be set to a string
// $filename:$lineno. If the location falls within the code lines of an
// interaction, the following happens:
//
//  1. Only the session that the interaction belongs to is run, and only up to
//     the located interaction.
//
//  2. If the actual output doesn't match what's in the file, the test fails,
//     and writes out a machine readable instruction to update the file to match
//     the actual output.
//
// As an example, consider the following fragment of foo_test.elvts (with line
// numbers):
//
//	12 ~> echo foo
//	13    echo bar
//	14 lorem
//	15 ipsum
//
// Running
//
//	env ELVISH_TRANSCRIPT_RUN=foo_test.elvts:12 go test -run TestTranscripts
//
// will end up with a test failure, with a message like the following (the line
// range is left-closed, right-open):
//
//	UPDATE {"fromLine": 14, "toLine": 16, "content": "foo\nbar\n"}
//
// This mechanism enables editor plugins that can fill or update the output of
// transcript tests without requiring user to leave the editor.
package evaltest

import (
	"bytes"
	"encoding/json"
	"fmt"
	"go/build/constraint"
	"io/fs"
	"math"
	"os"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"testing"

	"src.elv.sh/pkg/diag"
	"src.elv.sh/pkg/diff"
	"src.elv.sh/pkg/eval"
	"src.elv.sh/pkg/eval/vals"
	"src.elv.sh/pkg/mods"
	"src.elv.sh/pkg/must"
	"src.elv.sh/pkg/parse"
	"src.elv.sh/pkg/prog"
	"src.elv.sh/pkg/testutil"
	"src.elv.sh/pkg/transcript"
)

// TestTranscriptsInFS extracts all Elvish transcript sessions from .elv and
// .elvts files in fsys, and runs each of them as a test.
func TestTranscriptsInFS(t *testing.T, fsys fs.FS, setupPairs ...any) {
	nodes, err := transcript.ParseFromFS(fsys)
	if err != nil {
		t.Fatalf("parse transcript sessions: %v", err)
	}
	var run *runCfg
	if runEnv := os.Getenv("ELVISH_TRANSCRIPT_RUN"); runEnv != "" {
		filename, lineNo, ok := parseFileNameAndLineNo(runEnv)
		if !ok {
			t.Fatalf("can't parse ELVISH_TRANSCRIPT_RUN: %q", runEnv)
		}
		var node *transcript.Node
		for _, n := range nodes {
			if n.Name == filename {
				node = n
				break
			}
		}
		if node == nil {
			t.Fatalf("can't find file %q", filename)
		}
		nodes = []*transcript.Node{node}
		outputPrefix := ""
		if strings.HasSuffix(filename, ".elv") {
			outputPrefix = "# "
		}
		run = &runCfg{lineNo, outputPrefix}
	}
	testTranscripts(t, buildSetupDirectives(setupPairs), nodes, nil, run)
}

type runCfg struct {
	line         int
	outputPrefix string
}

func parseFileNameAndLineNo(s string) (string, int, bool) {
	i := strings.LastIndexByte(s, ':')
	if i == -1 {
		return "", 0, false
	}
	filename, lineNoString := s[:i], s[i+1:]
	lineNo, err := strconv.Atoi(lineNoString)
	if err != nil {
		return "", 0, false
	}
	return filename, lineNo, true
}

var solPattern = regexp.MustCompile("(?m:^)")

func testTranscripts(t *testing.T, sd *setupDirectives, nodes []*transcript.Node, setups []setupFunc, run *runCfg) {
	for _, node := range nodes {
		if run != nil && !(node.LineFrom <= run.line && run.line < node.LineTo) {
			continue
		}
		t.Run(node.Name, func(t *testing.T) {
			ev := eval.NewEvaler()
			mods.AddTo(ev)
			for _, setup := range setups {
				setup(t, ev)
			}
			var eachSetups []setupFunc
			for _, directive := range node.Directives {
				setup, each, err := sd.compile(directive)
				if err != nil {
					t.Fatal(err)
				}
				setup(t, ev)
				if each {
					eachSetups = append(eachSetups, setup)
				}
			}
			for _, interaction := range node.Interactions {
				if run != nil && interaction.CodeLineFrom > run.line {
					break
				}
				want := interaction.Output
				got := evalAndCollectOutput(ev, interaction.Code)
				if want != got {
					if run == nil {
						t.Errorf("\n%s\n-want +got:\n%s",
							interaction.PromptAndCode(), diff.DiffNoHeader(want, got))
					} else if interaction.CodeLineFrom <= run.line && run.line < interaction.CodeLineTo {
						content := got
						if run.outputPrefix != "" {
							// Insert output prefix at each SOL, except for the
							// SOL after the trailing newline.
							content = solPattern.ReplaceAllLiteralString(strings.TrimSuffix(content, "\n"), run.outputPrefix) + "\n"
						}
						correction := struct {
							FromLine int    `json:"fromLine"`
							ToLine   int    `json:"toLine"`
							Content  string `json:"content"`
						}{interaction.OutputLineFrom, interaction.OutputLineTo, content}
						t.Errorf("UPDATE %s", must.OK1(json.Marshal(correction)))
					}
				}
			}
			if len(node.Children) > 0 {
				// TODO: Use slices.Concat when Elvish requires Go 1.22
				allSetups := make([]setupFunc, 0, len(setups)+len(eachSetups))
				allSetups = append(allSetups, setups...)
				allSetups = append(allSetups, eachSetups...)
				testTranscripts(t, sd, node.Children, allSetups, run)
			}
		})
	}
}

type (
	setupFunc    func(*testing.T, *eval.Evaler)
	argSetupFunc func(*testing.T, *eval.Evaler, string)
)

type setupDirectives struct {
	setupMap    map[string]setupFunc
	argSetupMap map[string]argSetupFunc
}

func buildSetupDirectives(setupPairs []any) *setupDirectives {
	if len(setupPairs)%2 != 0 {
		panic(fmt.Sprintf("variadic arguments must come in pairs, got %d", len(setupPairs)))
	}
	setupMap := map[string]setupFunc{
		"in-temp-dir": func(t *testing.T, ev *eval.Evaler) { testutil.InTempDir(t) },
		"skip-test":   func(t *testing.T, _ *eval.Evaler) { t.SkipNow() },
	}
	argSetupMap := map[string]argSetupFunc{
		"set-env": func(t *testing.T, ev *eval.Evaler, arg string) {
			name, value, _ := strings.Cut(arg, " ")
			testutil.Setenv(t, name, value)
		},
		"unset-env": func(t *testing.T, ev *eval.Evaler, name string) {
			testutil.Unsetenv(t, name)
		},
		"eval": func(t *testing.T, ev *eval.Evaler, code string) {
			err := ev.Eval(
				parse.Source{Name: "[setup]", Code: code},
				eval.EvalCfg{Ports: eval.DummyPorts})
			if err != nil {
				t.Fatalf("setup failed: %v\n", err)
			}
		},
		"only-on": func(t *testing.T, _ *eval.Evaler, arg string) {
			expr, err := constraint.Parse("//go:build " + arg)
			if err != nil {
				t.Fatalf("parse constraint %q: %v", arg, err)
			}
			if !expr.Eval(func(tag string) bool {
				switch tag {
				case "unix":
					return isUNIX
				case "32bit":
					return math.MaxInt == math.MaxInt32
				case "64bit":
					return math.MaxInt == math.MaxInt64
				default:
					return tag == runtime.GOOS || tag == runtime.GOARCH
				}
			}) {
				t.Skipf("constraint not satisfied: %s", arg)
			}
		},
		"deprecation-level": func(t *testing.T, _ *eval.Evaler, arg string) {
			testutil.Set(t, &prog.DeprecationLevel, must.OK1(strconv.Atoi(arg)))
		},
	}
	for i := 0; i < len(setupPairs); i += 2 {
		name := setupPairs[i].(string)
		if setupMap[name] != nil || argSetupMap[name] != nil {
			panic(fmt.Sprintf("there's already a setup functions named %s", name))
		}
		switch f := setupPairs[i+1].(type) {
		case func():
			setupMap[name] = func(_ *testing.T, _ *eval.Evaler) { f() }
		case func(*testing.T):
			setupMap[name] = func(t *testing.T, ev *eval.Evaler) { f(t) }
		case func(*eval.Evaler):
			setupMap[name] = func(t *testing.T, ev *eval.Evaler) { f(ev) }
		case func(*testing.T, *eval.Evaler):
			setupMap[name] = f
		case func(string):
			argSetupMap[name] = func(_ *testing.T, _ *eval.Evaler, s string) { f(s) }
		case func(*testing.T, string):
			argSetupMap[name] = func(t *testing.T, _ *eval.Evaler, s string) { f(t, s) }
		case func(*eval.Evaler, string):
			argSetupMap[name] = func(_ *testing.T, ev *eval.Evaler, s string) { f(ev, s) }
		case func(*testing.T, *eval.Evaler, string):
			argSetupMap[name] = f
		default:
			panic(fmt.Sprintf("unsupported setup function type: %T", f))
		}
	}
	return &setupDirectives{setupMap, argSetupMap}
}

func (sd *setupDirectives) compile(directive string) (f setupFunc, each bool, err error) {
	cutDirective := directive
	if s, ok := strings.CutPrefix(directive, "each:"); ok {
		cutDirective = s
		each = true
	}
	name, arg, _ := strings.Cut(cutDirective, " ")
	if f, ok := sd.setupMap[name]; ok {
		if arg != "" {
			return nil, false, fmt.Errorf("setup function %q doesn't support arguments", name)
		}
		return f, each, nil
	} else if f, ok := sd.argSetupMap[name]; ok {
		return func(t *testing.T, ev *eval.Evaler) {
			f(t, ev, arg)
		}, each, nil
	} else {
		return nil, false, fmt.Errorf("unknown setup function %q in directive %q", name, directive)
	}
}

var valuePrefix = "▶ "

func evalAndCollectOutput(ev *eval.Evaler, code string) string {
	port1, collect1 := must.OK2(eval.CapturePort())
	port2, collect2 := must.OK2(eval.CapturePort())
	ports := []*eval.Port{eval.DummyInputPort, port1, port2}

	ctx, done := eval.ListenInterrupts()
	err := ev.Eval(
		parse.Source{Name: "[tty]", Code: code},
		eval.EvalCfg{Ports: ports, Interrupts: ctx})
	done()

	values, stdout := collect1()
	_, stderr := collect2()

	var sb strings.Builder
	for _, value := range values {
		sb.WriteString(valuePrefix + vals.ReprPlain(value) + "\n")
	}
	sb.Write(normalizeLineEnding(stripSGR(stdout)))
	sb.Write(normalizeLineEnding(stripSGR(stderr)))

	if err != nil {
		if shower, ok := err.(diag.Shower); ok {
			sb.WriteString(stripSGRString(shower.Show("")))
		} else {
			sb.WriteString(err.Error())
		}
		sb.WriteByte('\n')
	}

	return sb.String()
}

var sgrPattern = regexp.MustCompile("\033\\[[0-9;]*m")

func stripSGR(bs []byte) []byte      { return sgrPattern.ReplaceAllLiteral(bs, nil) }
func stripSGRString(s string) string { return sgrPattern.ReplaceAllLiteralString(s, "") }

func normalizeLineEnding(bs []byte) []byte { return bytes.ReplaceAll(bs, []byte("\r\n"), []byte("\n")) }