github.com/rajeev159/opa@v0.45.0/ast/internal/scanner/scanner_test.go (about)

     1  // Copyright 2020 The OPA Authors.  All rights reserved.
     2  // Use of this source code is governed by an Apache2
     3  // license that can be found in the LICENSE file.
     4  
     5  package scanner
     6  
     7  import (
     8  	"bytes"
     9  	"testing"
    10  
    11  	"github.com/open-policy-agent/opa/ast/internal/tokens"
    12  )
    13  
    14  func TestPositions(t *testing.T) {
    15  	tests := []struct {
    16  		note       string
    17  		input      string
    18  		wantOffset int
    19  		wantEnd    int
    20  	}{
    21  		{
    22  			note:       "symbol",
    23  			input:      "(",
    24  			wantOffset: 0,
    25  			wantEnd:    1,
    26  		},
    27  		{
    28  			note:       "ident",
    29  			input:      "foo",
    30  			wantOffset: 0,
    31  			wantEnd:    3,
    32  		},
    33  		{
    34  			note:       "number",
    35  			input:      "100",
    36  			wantOffset: 0,
    37  			wantEnd:    3,
    38  		},
    39  		{
    40  			note:       "string",
    41  			input:      `"foo"`,
    42  			wantOffset: 0,
    43  			wantEnd:    5,
    44  		},
    45  		{
    46  			note:       "string - wide char",
    47  			input:      `"foo÷"`,
    48  			wantOffset: 0,
    49  			wantEnd:    7,
    50  		},
    51  		{
    52  			note:       "comment",
    53  			input:      `# foo`,
    54  			wantOffset: 0,
    55  			wantEnd:    5,
    56  		},
    57  		{
    58  			note:       "newline",
    59  			input:      "foo\n",
    60  			wantOffset: 0,
    61  			wantEnd:    3,
    62  		},
    63  		{
    64  			note:       "invalid number",
    65  			input:      "0xDEADBEEF",
    66  			wantOffset: 0,
    67  			wantEnd:    10,
    68  		},
    69  		{
    70  			note:       "invalid identifier",
    71  			input:      "0.1e12a1b2c3d",
    72  			wantOffset: 0,
    73  			wantEnd:    13,
    74  		},
    75  	}
    76  
    77  	for _, tc := range tests {
    78  		t.Run(tc.note, func(t *testing.T) {
    79  			s, err := New(bytes.NewBufferString(tc.input))
    80  			if err != nil {
    81  				t.Fatal(err)
    82  			}
    83  			_, pos, _, _ := s.Scan()
    84  			if pos.Offset != tc.wantOffset {
    85  				t.Fatalf("want offset %d but got %d", tc.wantOffset, pos.Offset)
    86  			}
    87  			if pos.End != tc.wantEnd {
    88  				t.Fatalf("want end %d but got %d", tc.wantEnd, pos.End)
    89  			}
    90  		})
    91  	}
    92  }
    93  
    94  func TestLiterals(t *testing.T) {
    95  
    96  	tests := []struct {
    97  		note       string
    98  		input      string
    99  		wantRow    int
   100  		wantOffset int
   101  		wantTok    tokens.Token
   102  		wantLit    string
   103  	}{
   104  		{
   105  			note:       "ascii chars",
   106  			input:      `"hello world"`,
   107  			wantRow:    1,
   108  			wantOffset: 0,
   109  			wantTok:    tokens.String,
   110  			wantLit:    `"hello world"`,
   111  		},
   112  		{
   113  			note:       "wide chars",
   114  			input:      `"¡¡¡foo, bar!!!"`,
   115  			wantRow:    1,
   116  			wantOffset: 0,
   117  			wantTok:    tokens.String,
   118  			wantLit:    `"¡¡¡foo, bar!!!"`,
   119  		},
   120  		{
   121  			note:       "raw strings",
   122  			input:      "`foo`",
   123  			wantRow:    1,
   124  			wantOffset: 0,
   125  			wantTok:    tokens.String,
   126  			wantLit:    "`foo`",
   127  		},
   128  		{
   129  			note:       "raw strings - wide chars",
   130  			input:      "`¡¡¡foo, bar!!!`",
   131  			wantRow:    1,
   132  			wantOffset: 0,
   133  			wantTok:    tokens.String,
   134  			wantLit:    "`¡¡¡foo, bar!!!`",
   135  		},
   136  		{
   137  			note:       "comments",
   138  			input:      "# foo",
   139  			wantRow:    1,
   140  			wantOffset: 0,
   141  			wantTok:    tokens.Comment,
   142  			wantLit:    "# foo",
   143  		},
   144  		{
   145  			note:       "comments - wide chars",
   146  			input:      "#¡foo",
   147  			wantRow:    1,
   148  			wantOffset: 0,
   149  			wantTok:    tokens.Comment,
   150  			wantLit:    "#¡foo",
   151  		},
   152  	}
   153  
   154  	for _, tc := range tests {
   155  		t.Run(tc.note, func(t *testing.T) {
   156  			s, err := New(bytes.NewBufferString(tc.input))
   157  			if err != nil {
   158  				t.Fatal(err)
   159  			}
   160  			tok, pos, lit, errs := s.Scan()
   161  			if pos.Row != tc.wantRow {
   162  				t.Errorf("Expected row %d but got %d", tc.wantRow, pos.Row)
   163  			}
   164  			if pos.Offset != tc.wantOffset {
   165  				t.Errorf("Expected offset %d but got %d", tc.wantOffset, pos.Offset)
   166  			}
   167  			if tok != tc.wantTok {
   168  				t.Errorf("Expected token %v but got %v", tc.wantTok, tok)
   169  			}
   170  			if lit != tc.wantLit {
   171  				t.Errorf("Expected literal %v but got %v", tc.wantLit, lit)
   172  			}
   173  			if len(errs) > 0 {
   174  				t.Fatal("Unexpected error(s):", errs)
   175  			}
   176  		})
   177  	}
   178  
   179  }
   180  
   181  func TestIllegalTokens(t *testing.T) {
   182  
   183  	tests := []struct {
   184  		input   string
   185  		wantErr bool
   186  	}{
   187  		{input: `墳`},
   188  		{input: `0e`, wantErr: true},
   189  	}
   190  
   191  	for _, tc := range tests {
   192  		t.Run(tc.input, func(t *testing.T) {
   193  			s, err := New(bytes.NewBufferString(tc.input))
   194  			if err != nil {
   195  				t.Fatal(err)
   196  			}
   197  			tok, _, _, errs := s.Scan()
   198  			if !tc.wantErr && tok != tokens.Illegal {
   199  				t.Fatalf("expected illegal token on %q but got %v", tc.input, tok)
   200  			} else if tc.wantErr && len(errs) == 0 {
   201  				t.Fatalf("expected errors on %q but got %v", tc.input, tok)
   202  			}
   203  		})
   204  	}
   205  }