github.com/aretext/aretext@v1.3.0/syntax/languages/testutil.go (about)

     1  package languages
     2  
     3  import (
     4  	"errors"
     5  	"math"
     6  	"os"
     7  	"path/filepath"
     8  	"strings"
     9  	"testing"
    10  
    11  	"github.com/stretchr/testify/require"
    12  
    13  	"github.com/aretext/aretext/syntax/parser"
    14  	"github.com/aretext/aretext/text"
    15  )
    16  
    17  // TokenWithText is a token that includes its text value.
    18  type TokenWithText struct {
    19  	Role parser.TokenRole
    20  	Text string
    21  }
    22  
    23  // ParseTokensWithText tokenizes the input string using the specified parse func.
    24  func ParseTokensWithText(f parser.Func, s string) []TokenWithText {
    25  	p := parser.New(f)
    26  	tree, err := text.NewTreeFromString(s)
    27  	if err != nil {
    28  		panic(err)
    29  	}
    30  
    31  	stringSlice := func(startPos, endPos uint64) string {
    32  		var sb strings.Builder
    33  		reader := tree.ReaderAtPosition(startPos)
    34  		for i := startPos; i < endPos; i++ {
    35  			r, _, err := reader.ReadRune()
    36  			if err != nil {
    37  				break
    38  			}
    39  			_, err = sb.WriteRune(r)
    40  			if err != nil {
    41  				panic(err)
    42  			}
    43  		}
    44  		return sb.String()
    45  	}
    46  
    47  	p.ParseAll(tree)
    48  	tokens := p.TokensIntersectingRange(0, math.MaxUint64)
    49  	tokensWithText := make([]TokenWithText, 0, len(tokens))
    50  	for _, t := range tokens {
    51  		tokensWithText = append(tokensWithText, TokenWithText{
    52  			Role: t.Role,
    53  			Text: stringSlice(t.StartPos, t.EndPos),
    54  		})
    55  	}
    56  	return tokensWithText
    57  }
    58  
    59  // BenchmarkParser benchmarks a parser with the input file located at `path`.
    60  func BenchmarkParser(b *testing.B, f parser.Func, path string) {
    61  	data, err := os.ReadFile(path)
    62  	require.NoError(b, err)
    63  	tree, err := text.NewTreeFromString(string(data))
    64  	require.NoError(b, err)
    65  
    66  	p := parser.New(f)
    67  	for i := 0; i < b.N; i++ {
    68  		p.ParseAll(tree)
    69  	}
    70  }
    71  
    72  // FuzzParser runs a fuzz test on a parser.
    73  func FuzzParser(f *testing.F, parseFunc parser.Func, seeds []string) {
    74  	for _, seed := range seeds {
    75  		f.Add(seed)
    76  	}
    77  
    78  	f.Fuzz(func(t *testing.T, data string) {
    79  		tree, err := text.NewTreeFromString(data)
    80  		if errors.Is(err, text.ErrInvalidUtf8) {
    81  			t.Skip()
    82  		}
    83  		require.NoError(t, err)
    84  		p := parser.New(parseFunc)
    85  		p.ParseAll(tree)
    86  	})
    87  }
    88  
    89  // LoadFuzzTestSeeds loads seed data from files matching a glob pattern.
    90  func LoadFuzzTestSeeds(f *testing.F, globPattern string) []string {
    91  	var seeds []string
    92  
    93  	matches, err := filepath.Glob(globPattern)
    94  	if err != nil {
    95  		f.Fatalf("filepath.Glob: %s\n", err)
    96  	}
    97  
    98  	for _, path := range matches {
    99  		data, err := os.ReadFile(path)
   100  		if err != nil {
   101  			f.Fatalf("os.ReadFile: %s\n", err)
   102  		}
   103  
   104  		f.Logf("Loaded seed file %s\n", path)
   105  		seeds = append(seeds, string(data))
   106  	}
   107  
   108  	return seeds
   109  }