github.com/megatontech/mynoteforgo@v0.0.0-20200507084910-5d0c6ea6e890/源码/text/scanner/scanner_test.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package scanner 6 7 import ( 8 "bytes" 9 "fmt" 10 "io" 11 "strings" 12 "testing" 13 "unicode/utf8" 14 ) 15 16 // A StringReader delivers its data one string segment at a time via Read. 17 type StringReader struct { 18 data []string 19 step int 20 } 21 22 func (r *StringReader) Read(p []byte) (n int, err error) { 23 if r.step < len(r.data) { 24 s := r.data[r.step] 25 n = copy(p, s) 26 r.step++ 27 } else { 28 err = io.EOF 29 } 30 return 31 } 32 33 func readRuneSegments(t *testing.T, segments []string) { 34 got := "" 35 want := strings.Join(segments, "") 36 s := new(Scanner).Init(&StringReader{data: segments}) 37 for { 38 ch := s.Next() 39 if ch == EOF { 40 break 41 } 42 got += string(ch) 43 } 44 if got != want { 45 t.Errorf("segments=%v got=%s want=%s", segments, got, want) 46 } 47 } 48 49 var segmentList = [][]string{ 50 {}, 51 {""}, 52 {"日", "本語"}, 53 {"\u65e5", "\u672c", "\u8a9e"}, 54 {"\U000065e5", " ", "\U0000672c", "\U00008a9e"}, 55 {"\xe6", "\x97\xa5\xe6", "\x9c\xac\xe8\xaa\x9e"}, 56 {"Hello", ", ", "World", "!"}, 57 {"Hello", ", ", "", "World", "!"}, 58 } 59 60 func TestNext(t *testing.T) { 61 for _, s := range segmentList { 62 readRuneSegments(t, s) 63 } 64 } 65 66 type token struct { 67 tok rune 68 text string 69 } 70 71 var f100 = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" 72 73 var tokenList = []token{ 74 {Comment, "// line comments"}, 75 {Comment, "//"}, 76 {Comment, "////"}, 77 {Comment, "// comment"}, 78 {Comment, "// /* comment */"}, 79 {Comment, "// // comment //"}, 80 {Comment, "//" + f100}, 81 82 {Comment, "// general comments"}, 83 {Comment, "/**/"}, 84 {Comment, "/***/"}, 85 {Comment, "/* comment */"}, 86 {Comment, "/* // comment */"}, 87 {Comment, "/* /* comment */"}, 88 {Comment, "/*\n comment\n*/"}, 89 {Comment, "/*" + f100 + "*/"}, 90 91 {Comment, "// identifiers"}, 92 {Ident, "a"}, 93 {Ident, "a0"}, 94 {Ident, "foobar"}, 95 {Ident, "abc123"}, 96 {Ident, "LGTM"}, 97 {Ident, "_"}, 98 {Ident, "_abc123"}, 99 {Ident, "abc123_"}, 100 {Ident, "_abc_123_"}, 101 {Ident, "_äöü"}, 102 {Ident, "_本"}, 103 {Ident, "äöü"}, 104 {Ident, "本"}, 105 {Ident, "a۰۱۸"}, 106 {Ident, "foo६४"}, 107 {Ident, "bar9876"}, 108 {Ident, f100}, 109 110 {Comment, "// decimal ints"}, 111 {Int, "0"}, 112 {Int, "1"}, 113 {Int, "9"}, 114 {Int, "42"}, 115 {Int, "1234567890"}, 116 117 {Comment, "// octal ints"}, 118 {Int, "00"}, 119 {Int, "01"}, 120 {Int, "07"}, 121 {Int, "042"}, 122 {Int, "01234567"}, 123 124 {Comment, "// hexadecimal ints"}, 125 {Int, "0x0"}, 126 {Int, "0x1"}, 127 {Int, "0xf"}, 128 {Int, "0x42"}, 129 {Int, "0x123456789abcDEF"}, 130 {Int, "0x" + f100}, 131 {Int, "0X0"}, 132 {Int, "0X1"}, 133 {Int, "0XF"}, 134 {Int, "0X42"}, 135 {Int, "0X123456789abcDEF"}, 136 {Int, "0X" + f100}, 137 138 {Comment, "// floats"}, 139 {Float, "0."}, 140 {Float, "1."}, 141 {Float, "42."}, 142 {Float, "01234567890."}, 143 {Float, ".0"}, 144 {Float, ".1"}, 145 {Float, ".42"}, 146 {Float, ".0123456789"}, 147 {Float, "0.0"}, 148 {Float, "1.0"}, 149 {Float, "42.0"}, 150 {Float, "01234567890.0"}, 151 {Float, "0e0"}, 152 {Float, "1e0"}, 153 {Float, "42e0"}, 154 {Float, "01234567890e0"}, 155 {Float, "0E0"}, 156 {Float, "1E0"}, 157 {Float, "42E0"}, 158 {Float, "01234567890E0"}, 159 {Float, "0e+10"}, 160 {Float, "1e-10"}, 161 {Float, "42e+10"}, 162 {Float, "01234567890e-10"}, 163 {Float, "0E+10"}, 164 {Float, "1E-10"}, 165 {Float, "42E+10"}, 166 {Float, "01234567890E-10"}, 167 168 {Comment, "// chars"}, 169 {Char, `' '`}, 170 {Char, `'a'`}, 171 {Char, `'本'`}, 172 {Char, `'\a'`}, 173 {Char, `'\b'`}, 174 {Char, `'\f'`}, 175 {Char, `'\n'`}, 176 {Char, `'\r'`}, 177 {Char, `'\t'`}, 178 {Char, `'\v'`}, 179 {Char, `'\''`}, 180 {Char, `'\000'`}, 181 {Char, `'\777'`}, 182 {Char, `'\x00'`}, 183 {Char, `'\xff'`}, 184 {Char, `'\u0000'`}, 185 {Char, `'\ufA16'`}, 186 {Char, `'\U00000000'`}, 187 {Char, `'\U0000ffAB'`}, 188 189 {Comment, "// strings"}, 190 {String, `" "`}, 191 {String, `"a"`}, 192 {String, `"本"`}, 193 {String, `"\a"`}, 194 {String, `"\b"`}, 195 {String, `"\f"`}, 196 {String, `"\n"`}, 197 {String, `"\r"`}, 198 {String, `"\t"`}, 199 {String, `"\v"`}, 200 {String, `"\""`}, 201 {String, `"\000"`}, 202 {String, `"\777"`}, 203 {String, `"\x00"`}, 204 {String, `"\xff"`}, 205 {String, `"\u0000"`}, 206 {String, `"\ufA16"`}, 207 {String, `"\U00000000"`}, 208 {String, `"\U0000ffAB"`}, 209 {String, `"` + f100 + `"`}, 210 211 {Comment, "// raw strings"}, 212 {RawString, "``"}, 213 {RawString, "`\\`"}, 214 {RawString, "`" + "\n\n/* foobar */\n\n" + "`"}, 215 {RawString, "`" + f100 + "`"}, 216 217 {Comment, "// individual characters"}, 218 // NUL character is not allowed 219 {'\x01', "\x01"}, 220 {' ' - 1, string(' ' - 1)}, 221 {'+', "+"}, 222 {'/', "/"}, 223 {'.', "."}, 224 {'~', "~"}, 225 {'(', "("}, 226 } 227 228 func makeSource(pattern string) *bytes.Buffer { 229 var buf bytes.Buffer 230 for _, k := range tokenList { 231 fmt.Fprintf(&buf, pattern, k.text) 232 } 233 return &buf 234 } 235 236 func checkTok(t *testing.T, s *Scanner, line int, got, want rune, text string) { 237 if got != want { 238 t.Fatalf("tok = %s, want %s for %q", TokenString(got), TokenString(want), text) 239 } 240 if s.Line != line { 241 t.Errorf("line = %d, want %d for %q", s.Line, line, text) 242 } 243 stext := s.TokenText() 244 if stext != text { 245 t.Errorf("text = %q, want %q", stext, text) 246 } else { 247 // check idempotency of TokenText() call 248 stext = s.TokenText() 249 if stext != text { 250 t.Errorf("text = %q, want %q (idempotency check)", stext, text) 251 } 252 } 253 } 254 255 func checkTokErr(t *testing.T, s *Scanner, line int, want rune, text string) { 256 prevCount := s.ErrorCount 257 checkTok(t, s, line, s.Scan(), want, text) 258 if s.ErrorCount != prevCount+1 { 259 t.Fatalf("want error for %q", text) 260 } 261 } 262 263 func countNewlines(s string) int { 264 n := 0 265 for _, ch := range s { 266 if ch == '\n' { 267 n++ 268 } 269 } 270 return n 271 } 272 273 func testScan(t *testing.T, mode uint) { 274 s := new(Scanner).Init(makeSource(" \t%s\n")) 275 s.Mode = mode 276 tok := s.Scan() 277 line := 1 278 for _, k := range tokenList { 279 if mode&SkipComments == 0 || k.tok != Comment { 280 checkTok(t, s, line, tok, k.tok, k.text) 281 tok = s.Scan() 282 } 283 line += countNewlines(k.text) + 1 // each token is on a new line 284 } 285 checkTok(t, s, line, tok, EOF, "") 286 } 287 288 func TestScan(t *testing.T) { 289 testScan(t, GoTokens) 290 testScan(t, GoTokens&^SkipComments) 291 } 292 293 func TestIllegalExponent(t *testing.T) { 294 const src = "1.5e 1.5E 1e+ 1e- 1.5z" 295 s := new(Scanner).Init(strings.NewReader(src)) 296 checkTokErr(t, s, 1, Float, "1.5e") 297 checkTokErr(t, s, 1, Float, "1.5E") 298 checkTokErr(t, s, 1, Float, "1e+") 299 checkTokErr(t, s, 1, Float, "1e-") 300 checkTok(t, s, 1, s.Scan(), Float, "1.5") 301 checkTok(t, s, 1, s.Scan(), Ident, "z") 302 checkTok(t, s, 1, s.Scan(), EOF, "") 303 if s.ErrorCount != 4 { 304 t.Errorf("%d errors, want 4", s.ErrorCount) 305 } 306 } 307 308 func TestPosition(t *testing.T) { 309 src := makeSource("\t\t\t\t%s\n") 310 s := new(Scanner).Init(src) 311 s.Mode = GoTokens &^ SkipComments 312 s.Scan() 313 pos := Position{"", 4, 1, 5} 314 for _, k := range tokenList { 315 if s.Offset != pos.Offset { 316 t.Errorf("offset = %d, want %d for %q", s.Offset, pos.Offset, k.text) 317 } 318 if s.Line != pos.Line { 319 t.Errorf("line = %d, want %d for %q", s.Line, pos.Line, k.text) 320 } 321 if s.Column != pos.Column { 322 t.Errorf("column = %d, want %d for %q", s.Column, pos.Column, k.text) 323 } 324 pos.Offset += 4 + len(k.text) + 1 // 4 tabs + token bytes + newline 325 pos.Line += countNewlines(k.text) + 1 // each token is on a new line 326 s.Scan() 327 } 328 // make sure there were no token-internal errors reported by scanner 329 if s.ErrorCount != 0 { 330 t.Errorf("%d errors", s.ErrorCount) 331 } 332 } 333 334 func TestScanZeroMode(t *testing.T) { 335 src := makeSource("%s\n") 336 str := src.String() 337 s := new(Scanner).Init(src) 338 s.Mode = 0 // don't recognize any token classes 339 s.Whitespace = 0 // don't skip any whitespace 340 tok := s.Scan() 341 for i, ch := range str { 342 if tok != ch { 343 t.Fatalf("%d. tok = %s, want %s", i, TokenString(tok), TokenString(ch)) 344 } 345 tok = s.Scan() 346 } 347 if tok != EOF { 348 t.Fatalf("tok = %s, want EOF", TokenString(tok)) 349 } 350 if s.ErrorCount != 0 { 351 t.Errorf("%d errors", s.ErrorCount) 352 } 353 } 354 355 func testScanSelectedMode(t *testing.T, mode uint, class rune) { 356 src := makeSource("%s\n") 357 s := new(Scanner).Init(src) 358 s.Mode = mode 359 tok := s.Scan() 360 for tok != EOF { 361 if tok < 0 && tok != class { 362 t.Fatalf("tok = %s, want %s", TokenString(tok), TokenString(class)) 363 } 364 tok = s.Scan() 365 } 366 if s.ErrorCount != 0 { 367 t.Errorf("%d errors", s.ErrorCount) 368 } 369 } 370 371 func TestScanSelectedMask(t *testing.T) { 372 testScanSelectedMode(t, 0, 0) 373 testScanSelectedMode(t, ScanIdents, Ident) 374 // Don't test ScanInts and ScanNumbers since some parts of 375 // the floats in the source look like (illegal) octal ints 376 // and ScanNumbers may return either Int or Float. 377 testScanSelectedMode(t, ScanChars, Char) 378 testScanSelectedMode(t, ScanStrings, String) 379 testScanSelectedMode(t, SkipComments, 0) 380 testScanSelectedMode(t, ScanComments, Comment) 381 } 382 383 func TestScanCustomIdent(t *testing.T) { 384 const src = "faab12345 a12b123 a12 3b" 385 s := new(Scanner).Init(strings.NewReader(src)) 386 // ident = ( 'a' | 'b' ) { digit } . 387 // digit = '0' .. '3' . 388 // with a maximum length of 4 389 s.IsIdentRune = func(ch rune, i int) bool { 390 return i == 0 && (ch == 'a' || ch == 'b') || 0 < i && i < 4 && '0' <= ch && ch <= '3' 391 } 392 checkTok(t, s, 1, s.Scan(), 'f', "f") 393 checkTok(t, s, 1, s.Scan(), Ident, "a") 394 checkTok(t, s, 1, s.Scan(), Ident, "a") 395 checkTok(t, s, 1, s.Scan(), Ident, "b123") 396 checkTok(t, s, 1, s.Scan(), Int, "45") 397 checkTok(t, s, 1, s.Scan(), Ident, "a12") 398 checkTok(t, s, 1, s.Scan(), Ident, "b123") 399 checkTok(t, s, 1, s.Scan(), Ident, "a12") 400 checkTok(t, s, 1, s.Scan(), Int, "3") 401 checkTok(t, s, 1, s.Scan(), Ident, "b") 402 checkTok(t, s, 1, s.Scan(), EOF, "") 403 } 404 405 func TestScanNext(t *testing.T) { 406 const BOM = '\uFEFF' 407 BOMs := string(BOM) 408 s := new(Scanner).Init(strings.NewReader(BOMs + "if a == bcd /* com" + BOMs + "ment */ {\n\ta += c\n}" + BOMs + "// line comment ending in eof")) 409 checkTok(t, s, 1, s.Scan(), Ident, "if") // the first BOM is ignored 410 checkTok(t, s, 1, s.Scan(), Ident, "a") 411 checkTok(t, s, 1, s.Scan(), '=', "=") 412 checkTok(t, s, 0, s.Next(), '=', "") 413 checkTok(t, s, 0, s.Next(), ' ', "") 414 checkTok(t, s, 0, s.Next(), 'b', "") 415 checkTok(t, s, 1, s.Scan(), Ident, "cd") 416 checkTok(t, s, 1, s.Scan(), '{', "{") 417 checkTok(t, s, 2, s.Scan(), Ident, "a") 418 checkTok(t, s, 2, s.Scan(), '+', "+") 419 checkTok(t, s, 0, s.Next(), '=', "") 420 checkTok(t, s, 2, s.Scan(), Ident, "c") 421 checkTok(t, s, 3, s.Scan(), '}', "}") 422 checkTok(t, s, 3, s.Scan(), BOM, BOMs) 423 checkTok(t, s, 3, s.Scan(), -1, "") 424 if s.ErrorCount != 0 { 425 t.Errorf("%d errors", s.ErrorCount) 426 } 427 } 428 429 func TestScanWhitespace(t *testing.T) { 430 var buf bytes.Buffer 431 var ws uint64 432 // start at 1, NUL character is not allowed 433 for ch := byte(1); ch < ' '; ch++ { 434 buf.WriteByte(ch) 435 ws |= 1 << ch 436 } 437 const orig = 'x' 438 buf.WriteByte(orig) 439 440 s := new(Scanner).Init(&buf) 441 s.Mode = 0 442 s.Whitespace = ws 443 tok := s.Scan() 444 if tok != orig { 445 t.Errorf("tok = %s, want %s", TokenString(tok), TokenString(orig)) 446 } 447 } 448 449 func testError(t *testing.T, src, pos, msg string, tok rune) { 450 s := new(Scanner).Init(strings.NewReader(src)) 451 errorCalled := false 452 s.Error = func(s *Scanner, m string) { 453 if !errorCalled { 454 // only look at first error 455 if p := s.Pos().String(); p != pos { 456 t.Errorf("pos = %q, want %q for %q", p, pos, src) 457 } 458 if m != msg { 459 t.Errorf("msg = %q, want %q for %q", m, msg, src) 460 } 461 errorCalled = true 462 } 463 } 464 tk := s.Scan() 465 if tk != tok { 466 t.Errorf("tok = %s, want %s for %q", TokenString(tk), TokenString(tok), src) 467 } 468 if !errorCalled { 469 t.Errorf("error handler not called for %q", src) 470 } 471 if s.ErrorCount == 0 { 472 t.Errorf("count = %d, want > 0 for %q", s.ErrorCount, src) 473 } 474 } 475 476 func TestError(t *testing.T) { 477 testError(t, "\x00", "<input>:1:1", "illegal character NUL", 0) 478 testError(t, "\x80", "<input>:1:1", "illegal UTF-8 encoding", utf8.RuneError) 479 testError(t, "\xff", "<input>:1:1", "illegal UTF-8 encoding", utf8.RuneError) 480 481 testError(t, "a\x00", "<input>:1:2", "illegal character NUL", Ident) 482 testError(t, "ab\x80", "<input>:1:3", "illegal UTF-8 encoding", Ident) 483 testError(t, "abc\xff", "<input>:1:4", "illegal UTF-8 encoding", Ident) 484 485 testError(t, `"a`+"\x00", "<input>:1:3", "illegal character NUL", String) 486 testError(t, `"ab`+"\x80", "<input>:1:4", "illegal UTF-8 encoding", String) 487 testError(t, `"abc`+"\xff", "<input>:1:5", "illegal UTF-8 encoding", String) 488 489 testError(t, "`a"+"\x00", "<input>:1:3", "illegal character NUL", RawString) 490 testError(t, "`ab"+"\x80", "<input>:1:4", "illegal UTF-8 encoding", RawString) 491 testError(t, "`abc"+"\xff", "<input>:1:5", "illegal UTF-8 encoding", RawString) 492 493 testError(t, `'\"'`, "<input>:1:3", "illegal char escape", Char) 494 testError(t, `"\'"`, "<input>:1:3", "illegal char escape", String) 495 496 testError(t, `01238`, "<input>:1:6", "illegal octal number", Int) 497 testError(t, `01238123`, "<input>:1:9", "illegal octal number", Int) 498 testError(t, `0x`, "<input>:1:3", "illegal hexadecimal number", Int) 499 testError(t, `0xg`, "<input>:1:3", "illegal hexadecimal number", Int) 500 testError(t, `'aa'`, "<input>:1:4", "illegal char literal", Char) 501 testError(t, `1.5e`, "<input>:1:5", "illegal exponent", Float) 502 testError(t, `1.5E`, "<input>:1:5", "illegal exponent", Float) 503 testError(t, `1.5e+`, "<input>:1:6", "illegal exponent", Float) 504 testError(t, `1.5e-`, "<input>:1:6", "illegal exponent", Float) 505 506 testError(t, `'`, "<input>:1:2", "literal not terminated", Char) 507 testError(t, `'`+"\n", "<input>:1:2", "literal not terminated", Char) 508 testError(t, `"abc`, "<input>:1:5", "literal not terminated", String) 509 testError(t, `"abc`+"\n", "<input>:1:5", "literal not terminated", String) 510 testError(t, "`abc\n", "<input>:2:1", "literal not terminated", RawString) 511 testError(t, `/*/`, "<input>:1:4", "comment not terminated", EOF) 512 } 513 514 // An errReader returns (0, err) where err is not io.EOF. 515 type errReader struct{} 516 517 func (errReader) Read(b []byte) (int, error) { 518 return 0, io.ErrNoProgress // some error that is not io.EOF 519 } 520 521 func TestIOError(t *testing.T) { 522 s := new(Scanner).Init(errReader{}) 523 errorCalled := false 524 s.Error = func(s *Scanner, msg string) { 525 if !errorCalled { 526 if want := io.ErrNoProgress.Error(); msg != want { 527 t.Errorf("msg = %q, want %q", msg, want) 528 } 529 errorCalled = true 530 } 531 } 532 tok := s.Scan() 533 if tok != EOF { 534 t.Errorf("tok = %s, want EOF", TokenString(tok)) 535 } 536 if !errorCalled { 537 t.Errorf("error handler not called") 538 } 539 } 540 541 func checkPos(t *testing.T, got, want Position) { 542 if got.Offset != want.Offset || got.Line != want.Line || got.Column != want.Column { 543 t.Errorf("got offset, line, column = %d, %d, %d; want %d, %d, %d", 544 got.Offset, got.Line, got.Column, want.Offset, want.Line, want.Column) 545 } 546 } 547 548 func checkNextPos(t *testing.T, s *Scanner, offset, line, column int, char rune) { 549 if ch := s.Next(); ch != char { 550 t.Errorf("ch = %s, want %s", TokenString(ch), TokenString(char)) 551 } 552 want := Position{Offset: offset, Line: line, Column: column} 553 checkPos(t, s.Pos(), want) 554 } 555 556 func checkScanPos(t *testing.T, s *Scanner, offset, line, column int, char rune) { 557 want := Position{Offset: offset, Line: line, Column: column} 558 checkPos(t, s.Pos(), want) 559 if ch := s.Scan(); ch != char { 560 t.Errorf("ch = %s, want %s", TokenString(ch), TokenString(char)) 561 if string(ch) != s.TokenText() { 562 t.Errorf("tok = %q, want %q", s.TokenText(), string(ch)) 563 } 564 } 565 checkPos(t, s.Position, want) 566 } 567 568 func TestPos(t *testing.T) { 569 // corner case: empty source 570 s := new(Scanner).Init(strings.NewReader("")) 571 checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1}) 572 s.Peek() // peek doesn't affect the position 573 checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1}) 574 575 // corner case: source with only a newline 576 s = new(Scanner).Init(strings.NewReader("\n")) 577 checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1}) 578 checkNextPos(t, s, 1, 2, 1, '\n') 579 // after EOF position doesn't change 580 for i := 10; i > 0; i-- { 581 checkScanPos(t, s, 1, 2, 1, EOF) 582 } 583 if s.ErrorCount != 0 { 584 t.Errorf("%d errors", s.ErrorCount) 585 } 586 587 // corner case: source with only a single character 588 s = new(Scanner).Init(strings.NewReader("本")) 589 checkPos(t, s.Pos(), Position{Offset: 0, Line: 1, Column: 1}) 590 checkNextPos(t, s, 3, 1, 2, '本') 591 // after EOF position doesn't change 592 for i := 10; i > 0; i-- { 593 checkScanPos(t, s, 3, 1, 2, EOF) 594 } 595 if s.ErrorCount != 0 { 596 t.Errorf("%d errors", s.ErrorCount) 597 } 598 599 // positions after calling Next 600 s = new(Scanner).Init(strings.NewReader(" foo६४ \n\n本語\n")) 601 checkNextPos(t, s, 1, 1, 2, ' ') 602 s.Peek() // peek doesn't affect the position 603 checkNextPos(t, s, 2, 1, 3, ' ') 604 checkNextPos(t, s, 3, 1, 4, 'f') 605 checkNextPos(t, s, 4, 1, 5, 'o') 606 checkNextPos(t, s, 5, 1, 6, 'o') 607 checkNextPos(t, s, 8, 1, 7, '६') 608 checkNextPos(t, s, 11, 1, 8, '४') 609 checkNextPos(t, s, 12, 1, 9, ' ') 610 checkNextPos(t, s, 13, 1, 10, ' ') 611 checkNextPos(t, s, 14, 2, 1, '\n') 612 checkNextPos(t, s, 15, 3, 1, '\n') 613 checkNextPos(t, s, 18, 3, 2, '本') 614 checkNextPos(t, s, 21, 3, 3, '語') 615 checkNextPos(t, s, 22, 4, 1, '\n') 616 // after EOF position doesn't change 617 for i := 10; i > 0; i-- { 618 checkScanPos(t, s, 22, 4, 1, EOF) 619 } 620 if s.ErrorCount != 0 { 621 t.Errorf("%d errors", s.ErrorCount) 622 } 623 624 // positions after calling Scan 625 s = new(Scanner).Init(strings.NewReader("abc\n本語\n\nx")) 626 s.Mode = 0 627 s.Whitespace = 0 628 checkScanPos(t, s, 0, 1, 1, 'a') 629 s.Peek() // peek doesn't affect the position 630 checkScanPos(t, s, 1, 1, 2, 'b') 631 checkScanPos(t, s, 2, 1, 3, 'c') 632 checkScanPos(t, s, 3, 1, 4, '\n') 633 checkScanPos(t, s, 4, 2, 1, '本') 634 checkScanPos(t, s, 7, 2, 2, '語') 635 checkScanPos(t, s, 10, 2, 3, '\n') 636 checkScanPos(t, s, 11, 3, 1, '\n') 637 checkScanPos(t, s, 12, 4, 1, 'x') 638 // after EOF position doesn't change 639 for i := 10; i > 0; i-- { 640 checkScanPos(t, s, 13, 4, 2, EOF) 641 } 642 if s.ErrorCount != 0 { 643 t.Errorf("%d errors", s.ErrorCount) 644 } 645 } 646 647 type countReader int 648 649 func (r *countReader) Read([]byte) (int, error) { 650 *r++ 651 return 0, io.EOF 652 } 653 654 func TestNextEOFHandling(t *testing.T) { 655 var r countReader 656 657 // corner case: empty source 658 s := new(Scanner).Init(&r) 659 660 tok := s.Next() 661 if tok != EOF { 662 t.Error("1) EOF not reported") 663 } 664 665 tok = s.Peek() 666 if tok != EOF { 667 t.Error("2) EOF not reported") 668 } 669 670 if r != 1 { 671 t.Errorf("scanner called Read %d times, not once", r) 672 } 673 } 674 675 func TestScanEOFHandling(t *testing.T) { 676 var r countReader 677 678 // corner case: empty source 679 s := new(Scanner).Init(&r) 680 681 tok := s.Scan() 682 if tok != EOF { 683 t.Error("1) EOF not reported") 684 } 685 686 tok = s.Peek() 687 if tok != EOF { 688 t.Error("2) EOF not reported") 689 } 690 691 if r != 1 { 692 t.Errorf("scanner called Read %d times, not once", r) 693 } 694 }