github.com/megatontech/mynoteforgo@v0.0.0-20200507084910-5d0c6ea6e890/源码/text/scanner/scanner_test.go (about)

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package scanner
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"io"
    11  	"strings"
    12  	"testing"
    13  	"unicode/utf8"
    14  )
    15  
    16  // A StringReader delivers its data one string segment at a time via Read.
    17  type StringReader struct {
    18  	data []string
    19  	step int
    20  }
    21  
    22  func (r *StringReader) Read(p []byte) (n int, err error) {
    23  	if r.step < len(r.data) {
    24  		s := r.data[r.step]
    25  		n = copy(p, s)
    26  		r.step++
    27  	} else {
    28  		err = io.EOF
    29  	}
    30  	return
    31  }
    32  
    33  func readRuneSegments(t *testing.T, segments []string) {
    34  	got := ""
    35  	want := strings.Join(segments, "")
    36  	s := new(Scanner).Init(&StringReader{data: segments})
    37  	for {
    38  		ch := s.Next()
    39  		if ch == EOF {
    40  			break
    41  		}
    42  		got += string(ch)
    43  	}
    44  	if got != want {
    45  		t.Errorf("segments=%v got=%s want=%s", segments, got, want)
    46  	}
    47  }
    48  
    49  var segmentList = [][]string{
    50  	{},
    51  	{""},
    52  	{"日", "本語"},
    53  	{"\u65e5", "\u672c", "\u8a9e"},
    54  	{"\U000065e5", " ", "\U0000672c", "\U00008a9e"},
    55  	{"\xe6", "\x97\xa5\xe6", "\x9c\xac\xe8\xaa\x9e"},
    56  	{"Hello", ", ", "World", "!"},
    57  	{"Hello", ", ", "", "World", "!"},
    58  }
    59  
    60  func TestNext(t *testing.T) {
    61  	for _, s := range segmentList {
    62  		readRuneSegments(t, s)
    63  	}
    64  }
    65  
    66  type token struct {
    67  	tok  rune
    68  	text string
    69  }
    70  
    71  var f100 = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"
    72  
    73  var tokenList = []token{
    74  	{Comment, "// line comments"},
    75  	{Comment, "//"},
    76  	{Comment, "////"},
    77  	{Comment, "// comment"},
    78  	{Comment, "// /* comment */"},
    79  	{Comment, "// // comment //"},
    80  	{Comment, "//" + f100},
    81  
    82  	{Comment, "// general comments"},
    83  	{Comment, "/**/"},
    84  	{Comment, "/***/"},
    85  	{Comment, "/* comment */"},
    86  	{Comment, "/* // comment */"},
    87  	{Comment, "/* /* comment */"},
    88  	{Comment, "/*\n comment\n*/"},
    89  	{Comment, "/*" + f100 + "*/"},
    90  
    91  	{Comment, "// identifiers"},
    92  	{Ident, "a"},
    93  	{Ident, "a0"},
    94  	{Ident, "foobar"},
    95  	{Ident, "abc123"},
    96  	{Ident, "LGTM"},
    97  	{Ident, "_"},
    98  	{Ident, "_abc123"},
    99  	{Ident, "abc123_"},
   100  	{Ident, "_abc_123_"},
   101  	{Ident, "_äöü"},
   102  	{Ident, "_本"},
   103  	{Ident, "äöü"},
   104  	{Ident, "本"},
   105  	{Ident, "a۰۱۸"},
   106  	{Ident, "foo६४"},
   107  	{Ident, "bar9876"},
   108  	{Ident, f100},
   109  
   110  	{Comment, "// decimal ints"},
   111  	{Int, "0"},
   112  	{Int, "1"},
   113  	{Int, "9"},
   114  	{Int, "42"},
   115  	{Int, "1234567890"},
   116  
   117  	{Comment, "// octal ints"},
   118  	{Int, "00"},
   119  	{Int, "01"},
   120  	{Int, "07"},
   121  	{Int, "042"},
   122  	{Int, "01234567"},
   123  
   124  	{Comment, "// hexadecimal ints"},
   125  	{Int, "0x0"},
   126  	{Int, "0x1"},
   127  	{Int, "0xf"},
   128  	{Int, "0x42"},
   129  	{Int, "0x123456789abcDEF"},
   130  	{Int, "0x" + f100},
   131  	{Int, "0X0"},
   132  	{Int, "0X1"},
   133  	{Int, "0XF"},
   134  	{Int, "0X42"},
   135  	{Int, "0X123456789abcDEF"},
   136  	{Int, "0X" + f100},
   137  
   138  	{Comment, "// floats"},
   139  	{Float, "0."},
   140  	{Float, "1."},
   141  	{Float, "42."},
   142  	{Float, "01234567890."},
   143  	{Float, ".0"},
   144  	{Float, ".1"},
   145  	{Float, ".42"},
   146  	{Float, ".0123456789"},
   147  	{Float, "0.0"},
   148  	{Float, "1.0"},
   149  	{Float, "42.0"},
   150  	{Float, "01234567890.0"},
   151  	{Float, "0e0"},
   152  	{Float, "1e0"},
   153  	{Float, "42e0"},
   154  	{Float, "01234567890e0"},
   155  	{Float, "0E0"},
   156  	{Float, "1E0"},
   157  	{Float, "42E0"},
   158  	{Float, "01234567890E0"},
   159  	{Float, "0e+10"},
   160  	{Float, "1e-10"},
   161  	{Float, "42e+10"},
   162  	{Float, "01234567890e-10"},
   163  	{Float, "0E+10"},
   164  	{Float, "1E-10"},
   165  	{Float, "42E+10"},
   166  	{Float, "01234567890E-10"},
   167  
   168  	{Comment, "// chars"},
   169  	{Char, `' '`},
   170  	{Char, `'a'`},
   171  	{Char, `'本'`},
   172  	{Char, `'\a'`},
   173  	{Char, `'\b'`},
   174  	{Char, `'\f'`},
   175  	{Char, `'\n'`},
   176  	{Char, `'\r'`},
   177  	{Char, `'\t'`},
   178  	{Char, `'\v'`},
   179  	{Char, `'\''`},
   180  	{Char, `'\000'`},
   181  	{Char, `'\777'`},
   182  	{Char, `'\x00'`},
   183  	{Char, `'\xff'`},
   184  	{Char, `'\u0000'`},
   185  	{Char, `'\ufA16'`},
   186  	{Char, `'\U00000000'`},
   187  	{Char, `'\U0000ffAB'`},
   188  
   189  	{Comment, "// strings"},
   190  	{String, `" "`},
   191  	{String, `"a"`},
   192  	{String, `"本"`},
   193  	{String, `"\a"`},
   194  	{String, `"\b"`},
   195  	{String, `"\f"`},
   196  	{String, `"\n"`},
   197  	{String, `"\r"`},
   198  	{String, `"\t"`},
   199  	{String, `"\v"`},
   200  	{String, `"\""`},
   201  	{String, `"\000"`},
   202  	{String, `"\777"`},
   203  	{String, `"\x00"`},
   204  	{String, `"\xff"`},
   205  	{String, `"\u0000"`},
   206  	{String, `"\ufA16"`},
   207  	{String, `"\U00000000"`},
   208  	{String, `"\U0000ffAB"`},
   209  	{String, `"` + f100 + `"`},
   210  
   211  	{Comment, "// raw strings"},
   212  	{RawString, "``"},
   213  	{RawString, "`\\`"},
   214  	{RawString, "`" + "\n\n/* foobar */\n\n" + "`"},
   215  	{RawString, "`" + f100 + "`"},
   216  
   217  	{Comment, "// individual characters"},
   218  	// NUL character is not allowed
   219  	{'\x01', "\x01"},
   220  	{' ' - 1, string(' ' - 1)},
   221  	{'+', "+"},
   222  	{'/', "/"},
   223  	{'.', "."},
   224  	{'~', "~"},
   225  	{'(', "("},
   226  }
   227  
   228  func makeSource(pattern string) *bytes.Buffer {
   229  	var buf bytes.Buffer
   230  	for _, k := range tokenList {
   231  		fmt.Fprintf(&buf, pattern, k.text)
   232  	}
   233  	return &buf
   234  }
   235  
   236  func checkTok(t *testing.T, s *Scanner, line int, got, want rune, text string) {
   237  	if got != want {
   238  		t.Fatalf("tok = %s, want %s for %q", TokenString(got), TokenString(want), text)
   239  	}
   240  	if s.Line != line {
   241  		t.Errorf("line = %d, want %d for %q", s.Line, line, text)
   242  	}
   243  	stext := s.TokenText()
   244  	if stext != text {
   245  		t.Errorf("text = %q, want %q", stext, text)
   246  	} else {
   247  		// check idempotency of TokenText() call
   248  		stext = s.TokenText()
   249  		if stext != text {
   250  			t.Errorf("text = %q, want %q (idempotency check)", stext, text)
   251  		}
   252  	}
   253  }
   254  
   255  func checkTokErr(t *testing.T, s *Scanner, line int, want rune, text string) {
   256  	prevCount := s.ErrorCount
   257  	checkTok(t, s, line, s.Scan(), want, text)
   258  	if s.ErrorCount != prevCount+1 {
   259  		t.Fatalf("want error for %q", text)
   260  	}
   261  }
   262  
   263  func countNewlines(s string) int {
   264  	n := 0
   265  	for _, ch := range s {
   266  		if ch == '\n' {
   267  			n++
   268  		}
   269  	}
   270  	return n
   271  }
   272  
   273  func testScan(t *testing.T, mode uint) {
   274  	s := new(Scanner).Init(makeSource(" \t%s\n"))
   275  	s.Mode = mode
   276  	tok := s.Scan()
   277  	line := 1
   278  	for _, k := range tokenList {
   279  		if mode&SkipComments == 0 || k.tok != Comment {
   280  			checkTok(t, s, line, tok, k.tok, k.text)
   281  			tok = s.Scan()
   282  		}
   283  		line += countNewlines(k.text) + 1 // each token is on a new line
   284  	}
   285  	checkTok(t, s, line, tok, EOF, "")
   286  }
   287  
   288  func TestScan(t *testing.T) {
   289  	testScan(t, GoTokens)
   290  	testScan(t, GoTokens&^SkipComments)
   291  }
   292  
   293  func TestIllegalExponent(t *testing.T) {
   294  	const src = "1.5e 1.5E 1e+ 1e- 1.5z"
   295  	s := new(Scanner).Init(strings.NewReader(src))
   296  	checkTokErr(t, s, 1, Float, "1.5e")
   297  	checkTokErr(t, s, 1, Float, "1.5E")
   298  	checkTokErr(t, s, 1, Float, "1e+")
   299  	checkTokErr(t, s, 1, Float, "1e-")
   300  	checkTok(t, s, 1, s.Scan(), Float, "1.5")
   301  	checkTok(t, s, 1, s.Scan(), Ident, "z")
   302  	checkTok(t, s, 1, s.Scan(), EOF, "")
   303  	if s.ErrorCount != 4 {
   304  		t.Errorf("%d errors, want 4", s.ErrorCount)
   305  	}
   306  }
   307  
   308  func TestPosition(t *testing.T) {
   309  	src := makeSource("\t\t\t\t%s\n")
   310  	s := new(Scanner).Init(src)
   311  	s.Mode = GoTokens &^ SkipComments
   312  	s.Scan()
   313  	pos := Position{"", 4, 1, 5}
   314  	for _, k := range tokenList {
   315  		if s.Offset != pos.Offset {
   316  			t.Errorf("offset = %d, want %d for %q", s.Offset, pos.Offset, k.text)
   317  		}
   318  		if s.Line != pos.Line {
   319  			t.Errorf("line = %d, want %d for %q", s.Line, pos.Line, k.text)
   320  		}
   321  		if s.Column != pos.Column {
   322  			t.Errorf("column = %d, want %d for %q", s.Column, pos.Column, k.text)
   323  		}
   324  		pos.Offset += 4 + len(k.text) + 1     // 4 tabs + token bytes + newline
   325  		pos.Line += countNewlines(k.text) + 1 // each token is on a new line
   326  		s.Scan()
   327  	}
   328  	// make sure there were no token-internal errors reported by scanner
   329  	if s.ErrorCount != 0 {
   330  		t.Errorf("%d errors", s.ErrorCount)
   331  	}
   332  }
   333  
   334  func TestScanZeroMode(t *testing.T) {
   335  	src := makeSource("%s\n")
   336  	str := src.String()
   337  	s := new(Scanner).Init(src)
   338  	s.Mode = 0       // don't recognize any token classes
   339  	s.Whitespace = 0 // don't skip any whitespace
   340  	tok := s.Scan()
   341  	for i, ch := range str {
   342  		if tok != ch {
   343  			t.Fatalf("%d. tok = %s, want %s", i, TokenString(tok), TokenString(ch))
   344  		}
   345  		tok = s.Scan()
   346  	}
   347  	if tok != EOF {
   348  		t.Fatalf("tok = %s, want EOF", TokenString(tok))
   349  	}
   350  	if s.ErrorCount != 0 {
   351  		t.Errorf("%d errors", s.ErrorCount)
   352  	}
   353  }
   354  
   355  func testScanSelectedMode(t *testing.T, mode uint, class rune) {
   356  	src := makeSource("%s\n")
   357  	s := new(Scanner).Init(src)
   358  	s.Mode = mode
   359  	tok := s.Scan()
   360  	for tok != EOF {
   361  		if tok < 0 && tok != class {
   362  			t.Fatalf("tok = %s, want %s", TokenString(tok), TokenString(class))
   363  		}
   364  		tok = s.Scan()
   365  	}
   366  	if s.ErrorCount != 0 {
   367  		t.Errorf("%d errors", s.ErrorCount)
   368  	}
   369  }
   370  
   371  func TestScanSelectedMask(t *testing.T) {
   372  	testScanSelectedMode(t, 0, 0)
   373  	testScanSelectedMode(t, ScanIdents, Ident)
   374  	// Don't test ScanInts and ScanNumbers since some parts of
   375  	// the floats in the source look like (illegal) octal ints
   376  	// and ScanNumbers may return either Int or Float.
   377  	testScanSelectedMode(t, ScanChars, Char)
   378  	testScanSelectedMode(t, ScanStrings, String)
   379  	testScanSelectedMode(t, SkipComments, 0)
   380  	testScanSelectedMode(t, ScanComments, Comment)
   381  }
   382  
   383  func TestScanCustomIdent(t *testing.T) {
   384  	const src = "faab12345 a12b123 a12 3b"
   385  	s := new(Scanner).Init(strings.NewReader(src))
   386  	// ident = ( 'a' | 'b' ) { digit } .
   387  	// digit = '0' .. '3' .
   388  	// with a maximum length of 4
   389  	s.IsIdentRune = func(ch rune, i int) bool {
   390  		return i == 0 && (ch == 'a' || ch == 'b') || 0 < i && i < 4 && '0' <= ch && ch <= '3'
   391  	}
   392  	checkTok(t, s, 1, s.Scan(), 'f', "f")
   393  	checkTok(t, s, 1, s.Scan(), Ident, "a")
   394  	checkTok(t, s, 1, s.Scan(), Ident, "a")
   395  	checkTok(t, s, 1, s.Scan(), Ident, "b123")
   396  	checkTok(t, s, 1, s.Scan(), Int, "45")
   397  	checkTok(t, s, 1, s.Scan(), Ident, "a12")
   398  	checkTok(t, s, 1, s.Scan(), Ident, "b123")
   399  	checkTok(t, s, 1, s.Scan(), Ident, "a12")
   400  	checkTok(t, s, 1, s.Scan(), Int, "3")
   401  	checkTok(t, s, 1, s.Scan(), Ident, "b")
   402  	checkTok(t, s, 1, s.Scan(), EOF, "")
   403  }
   404  
   405  func TestScanNext(t *testing.T) {
   406  	const BOM = '\uFEFF'
   407  	BOMs := string(BOM)
   408  	s := new(Scanner).Init(strings.NewReader(BOMs + "if a == bcd /* com" + BOMs + "ment */ {\n\ta += c\n}" + BOMs + "// line comment ending in eof"))
   409  	checkTok(t, s, 1, s.Scan(), Ident, "if") // the first BOM is ignored
   410  	checkTok(t, s, 1, s.Scan(), Ident, "a")
   411  	checkTok(t, s, 1, s.Scan(), '=', "=")
   412  	checkTok(t, s, 0, s.Next(), '=', "")
   413  	checkTok(t, s, 0, s.Next(), ' ', "")
   414  	checkTok(t, s, 0, s.Next(), 'b', "")
   415  	checkTok(t, s, 1, s.Scan(), Ident, "cd")
   416  	checkTok(t, s, 1, s.Scan(), '{', "{")
   417  	checkTok(t, s, 2, s.Scan(), Ident, "a")
   418  	checkTok(t, s, 2, s.Scan(), '+', "+")
   419  	checkTok(t, s, 0, s.Next(), '=', "")
   420  	checkTok(t, s, 2, s.Scan(), Ident, "c")
   421  	checkTok(t, s, 3, s.Scan(), '}', "}")
   422  	checkTok(t, s, 3, s.Scan(), BOM, BOMs)
   423  	checkTok(t, s, 3, s.Scan(), -1, "")
   424  	if s.ErrorCount != 0 {
   425  		t.Errorf("%d errors", s.ErrorCount)
   426  	}
   427  }
   428  
   429  func TestScanWhitespace(t *testing.T) {
   430  	var buf bytes.Buffer
   431  	var ws uint64
   432  	// start at 1, NUL character is not allowed
   433  	for ch := byte(1); ch < ' '; ch++ {
   434  		buf.WriteByte(ch)
   435  		ws |= 1 << ch
   436  	}
   437  	const orig = 'x'
   438  	buf.WriteByte(orig)
   439  
   440  	s := new(Scanner).Init(&buf)
   441  	s.Mode = 0
   442  	s.Whitespace = ws
   443  	tok := s.Scan()
   444  	if tok != orig {
   445  		t.Errorf("tok = %s, want %s", TokenString(tok), TokenString(orig))
   446  	}
   447  }
   448  
   449  func testError(t *testing.T, src, pos, msg string, tok rune) {
   450  	s := new(Scanner).Init(strings.NewReader(src))
   451  	errorCalled := false
   452  	s.Error = func(s *Scanner, m string) {
   453  		if !errorCalled {
   454  			// only look at first error
   455  			if p := s.Pos().String(); p != pos {
   456  				t.Errorf("pos = %q, want %q for %q", p, pos, src)
   457  			}
   458  			if m != msg {
   459  				t.Errorf("msg = %q, want %q for %q", m, msg, src)
   460  			}
   461  			errorCalled = true
   462  		}
   463  	}
   464  	tk := s.Scan()
   465  	if tk != tok {
   466  		t.Errorf("tok = %s, want %s for %q", TokenString(tk), TokenString(tok), src)
   467  	}
   468  	if !errorCalled {
   469  		t.Errorf("error handler not called for %q", src)
   470  	}
   471  	if s.ErrorCount == 0 {
   472  		t.Errorf("count = %d, want > 0 for %q", s.ErrorCount, src)
   473  	}
   474  }
   475  
   476  func TestError(t *testing.T) {
   477  	testError(t, "\x00", "<input>:1:1", "illegal character NUL", 0)
   478  	testError(t, "\x80", "<input>:1:1", "illegal UTF-8 encoding", utf8.RuneError)
   479  	testError(t, "\xff", "<input>:1:1", "illegal UTF-8 encoding", utf8.RuneError)
   480  
   481  	testError(t, "a\x00", "<input>:1:2", "illegal character NUL", Ident)
   482  	testError(t, "ab\x80", "<input>:1:3", "illegal UTF-8 encoding", Ident)
   483  	testError(t, "abc\xff", "<input>:1:4", "illegal UTF-8 encoding", Ident)
   484  
   485  	testError(t, `"a`+"\x00", "<input>:1:3", "illegal character NUL", String)
   486  	testError(t, `"ab`+"\x80", "<input>:1:4", "illegal UTF-8 encoding", String)
   487  	testError(t, `"abc`+"\xff", "<input>:1:5", "illegal UTF-8 encoding", String)
   488  
   489  	testError(t, "`a"+"\x00", "<input>:1:3", "illegal character NUL", RawString)
   490  	testError(t, "`ab"+"\x80", "<input>:1:4", "illegal UTF-8 encoding", RawString)
   491  	testError(t, "`abc"+"\xff", "<input>:1:5", "illegal UTF-8 encoding", RawString)
   492  
   493  	testError(t, `'\"'`, "<input>:1:3", "illegal char escape", Char)
   494  	testError(t, `"\'"`, "<input>:1:3", "illegal char escape", String)
   495  
   496  	testError(t, `01238`, "<input>:1:6", "illegal octal number", Int)
   497  	testError(t, `01238123`, "<input>:1:9", "illegal octal number", Int)
   498  	testError(t, `0x`, "<input>:1:3", "illegal hexadecimal number", Int)
   499  	testError(t, `0xg`, "<input>:1:3", "illegal hexadecimal number", Int)
   500  	testError(t, `'aa'`, "<input>:1:4", "illegal char literal", Char)
   501  	testError(t, `1.5e`, "<input>:1:5", "illegal exponent", Float)
   502  	testError(t, `1.5E`, "<input>:1:5", "illegal exponent", Float)
   503  	testError(t, `1.5e+`, "<input>:1:6", "illegal exponent", Float)
   504  	testError(t, `1.5e-`, "<input>:1:6", "illegal exponent", Float)
   505  
   506  	testError(t, `'`, "<input>:1:2", "literal not terminated", Char)
   507  	testError(t, `'`+"\n", "<input>:1:2", "literal not terminated", Char)
   508  	testError(t, `"abc`, "<input>:1:5", "literal not terminated", String)
   509  	testError(t, `"abc`+"\n", "<input>:1:5", "literal not terminated", String)
   510  	testError(t, "`abc\n", "<input>:2:1", "literal not terminated", RawString)
   511  	testError(t, `/*/`, "<input>:1:4", "comment not terminated", EOF)
   512  }
   513  
   514  // An errReader returns (0, err) where err is not io.EOF.
   515  type errReader struct{}
   516  
   517  func (errReader) Read(b []byte) (int, error) {
   518  	return 0, io.ErrNoProgress // some error that is not io.EOF
   519  }
   520  
   521  func TestIOError(t *testing.T) {
   522  	s := new(Scanner).Init(errReader{})
   523  	errorCalled := false
   524  	s.Error = func(s *Scanner, msg string) {
   525  		if !errorCalled {
   526  			if want := io.ErrNoProgress.Error(); msg != want {
   527  				t.Errorf("msg = %q, want %q", msg, want)
   528  			}
   529  			errorCalled = true
   530  		}
   531  	}
   532  	tok := s.Scan()
   533  	if tok != EOF {
   534  		t.Errorf("tok = %s, want EOF", TokenString(tok))
   535  	}
   536  	if !errorCalled {
   537  		t.Errorf("error handler not called")
   538  	}
   539  }
   540  
   541  func checkPos(t *testing.T, got, want Position) {
   542  	if got.Offset != want.Offset || got.Line != want.Line || got.Column != want.Column {
   543  		t.Errorf("got offset, line, column = %d, %d, %d; want %d, %d, %d",
   544  			got.Offset, got.Line, got.Column, want.Offset, want.Line, want.Column)
   545  	}
   546  }
   547  
   548  func checkNextPos(t *testing.T, s *Scanner, offset, line, column int, char rune) {
   549  	if ch := s.Next(); ch != char {
   550  		t.Errorf("ch = %s, want %s", TokenString(ch), TokenString(char))
   551  	}
   552  	want := Position{Offset: offset, Line: line, Column: column}
   553  	checkPos(t, s.Pos(), want)
   554  }
   555  
   556  func checkScanPos(t *testing.T, s *Scanner, offset, line, column int, char rune) {
   557  	want := Position{Offset: offset, Line: line, Column: column}
   558  	checkPos(t, s.Pos(), want)
   559  	if ch := s.Scan(); ch != char {
   560  		t.Errorf("ch = %s, want %s", TokenString(ch), TokenString(char))
   561  		if string(ch) != s.TokenText() {
   562  			t.Errorf("tok = %q, want %q", s.TokenText(), string(ch))
   563  		}
   564  	}
   565  	checkPos(t, s.Position, want)
   566  }
   567  
   568  func TestPos(t *testing.T) {
   569  	// corner case: empty source
   570  	s := new(Scanner).Init(strings.NewReader(""))
   571  	checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1})
   572  	s.Peek() // peek doesn't affect the position
   573  	checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1})
   574  
   575  	// corner case: source with only a newline
   576  	s = new(Scanner).Init(strings.NewReader("\n"))
   577  	checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1})
   578  	checkNextPos(t, s, 1, 2, 1, '\n')
   579  	// after EOF position doesn't change
   580  	for i := 10; i > 0; i-- {
   581  		checkScanPos(t, s, 1, 2, 1, EOF)
   582  	}
   583  	if s.ErrorCount != 0 {
   584  		t.Errorf("%d errors", s.ErrorCount)
   585  	}
   586  
   587  	// corner case: source with only a single character
   588  	s = new(Scanner).Init(strings.NewReader("本"))
   589  	checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1})
   590  	checkNextPos(t, s, 3, 1, 2, '本')
   591  	// after EOF position doesn't change
   592  	for i := 10; i > 0; i-- {
   593  		checkScanPos(t, s, 3, 1, 2, EOF)
   594  	}
   595  	if s.ErrorCount != 0 {
   596  		t.Errorf("%d errors", s.ErrorCount)
   597  	}
   598  
   599  	// positions after calling Next
   600  	s = new(Scanner).Init(strings.NewReader("  foo६४  \n\n本語\n"))
   601  	checkNextPos(t, s, 1, 1, 2, ' ')
   602  	s.Peek() // peek doesn't affect the position
   603  	checkNextPos(t, s, 2, 1, 3, ' ')
   604  	checkNextPos(t, s, 3, 1, 4, 'f')
   605  	checkNextPos(t, s, 4, 1, 5, 'o')
   606  	checkNextPos(t, s, 5, 1, 6, 'o')
   607  	checkNextPos(t, s, 8, 1, 7, '६')
   608  	checkNextPos(t, s, 11, 1, 8, '४')
   609  	checkNextPos(t, s, 12, 1, 9, ' ')
   610  	checkNextPos(t, s, 13, 1, 10, ' ')
   611  	checkNextPos(t, s, 14, 2, 1, '\n')
   612  	checkNextPos(t, s, 15, 3, 1, '\n')
   613  	checkNextPos(t, s, 18, 3, 2, '本')
   614  	checkNextPos(t, s, 21, 3, 3, '語')
   615  	checkNextPos(t, s, 22, 4, 1, '\n')
   616  	// after EOF position doesn't change
   617  	for i := 10; i > 0; i-- {
   618  		checkScanPos(t, s, 22, 4, 1, EOF)
   619  	}
   620  	if s.ErrorCount != 0 {
   621  		t.Errorf("%d errors", s.ErrorCount)
   622  	}
   623  
   624  	// positions after calling Scan
   625  	s = new(Scanner).Init(strings.NewReader("abc\n本語\n\nx"))
   626  	s.Mode = 0
   627  	s.Whitespace = 0
   628  	checkScanPos(t, s, 0, 1, 1, 'a')
   629  	s.Peek() // peek doesn't affect the position
   630  	checkScanPos(t, s, 1, 1, 2, 'b')
   631  	checkScanPos(t, s, 2, 1, 3, 'c')
   632  	checkScanPos(t, s, 3, 1, 4, '\n')
   633  	checkScanPos(t, s, 4, 2, 1, '本')
   634  	checkScanPos(t, s, 7, 2, 2, '語')
   635  	checkScanPos(t, s, 10, 2, 3, '\n')
   636  	checkScanPos(t, s, 11, 3, 1, '\n')
   637  	checkScanPos(t, s, 12, 4, 1, 'x')
   638  	// after EOF position doesn't change
   639  	for i := 10; i > 0; i-- {
   640  		checkScanPos(t, s, 13, 4, 2, EOF)
   641  	}
   642  	if s.ErrorCount != 0 {
   643  		t.Errorf("%d errors", s.ErrorCount)
   644  	}
   645  }
   646  
   647  type countReader int
   648  
   649  func (r *countReader) Read([]byte) (int, error) {
   650  	*r++
   651  	return 0, io.EOF
   652  }
   653  
   654  func TestNextEOFHandling(t *testing.T) {
   655  	var r countReader
   656  
   657  	// corner case: empty source
   658  	s := new(Scanner).Init(&r)
   659  
   660  	tok := s.Next()
   661  	if tok != EOF {
   662  		t.Error("1) EOF not reported")
   663  	}
   664  
   665  	tok = s.Peek()
   666  	if tok != EOF {
   667  		t.Error("2) EOF not reported")
   668  	}
   669  
   670  	if r != 1 {
   671  		t.Errorf("scanner called Read %d times, not once", r)
   672  	}
   673  }
   674  
   675  func TestScanEOFHandling(t *testing.T) {
   676  	var r countReader
   677  
   678  	// corner case: empty source
   679  	s := new(Scanner).Init(&r)
   680  
   681  	tok := s.Scan()
   682  	if tok != EOF {
   683  		t.Error("1) EOF not reported")
   684  	}
   685  
   686  	tok = s.Peek()
   687  	if tok != EOF {
   688  		t.Error("2) EOF not reported")
   689  	}
   690  
   691  	if r != 1 {
   692  		t.Errorf("scanner called Read %d times, not once", r)
   693  	}
   694  }