github.com/influx6/npkg@v0.8.8/ntext/nscanner/example_test.go (about)

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package nscanner_test
     6  
     7  import (
     8  	"fmt"
     9  	"strings"
    10  	"unicode"
    11  
    12  	scanner "github.com/influx6/npkg/ntext/nscanner"
    13  )
    14  
    15  func Example() {
    16  	const src = `
    17  // This is scanned code.
    18  if a > 10 {
    19  	someParsable = text
    20  }`
    21  
    22  	var s scanner.Scanner
    23  	s.Init(strings.NewReader(src))
    24  	s.Filename = "example"
    25  	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
    26  		fmt.Printf("%s: %s\n", s.Position, s.TokenText())
    27  	}
    28  
    29  	// Output:
    30  	// example:3:1: if
    31  	// example:3:4: a
    32  	// example:3:6: >
    33  	// example:3:8: 10
    34  	// example:3:11: {
    35  	// example:4:2: someParsable
    36  	// example:4:15: =
    37  	// example:4:17: text
    38  	// example:5:1: }
    39  }
    40  
    41  func Example_isIdentRune() {
    42  	const src = "%var1 var2%"
    43  
    44  	var s scanner.Scanner
    45  	s.Init(strings.NewReader(src))
    46  	s.Filename = "default"
    47  
    48  	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
    49  		fmt.Printf("%s: %s\n", s.Position, s.TokenText())
    50  	}
    51  
    52  	fmt.Println()
    53  	s.Init(strings.NewReader(src))
    54  	s.Filename = "percent"
    55  
    56  	// treat leading '%' as part of an identifier
    57  	s.IsIdentRune = func(ch rune, i int) bool {
    58  		return ch == '%' && i == 0 || unicode.IsLetter(ch) || unicode.IsDigit(ch) && i > 0
    59  	}
    60  
    61  	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
    62  		fmt.Printf("%s: %s\n", s.Position, s.TokenText())
    63  	}
    64  
    65  	// Output:
    66  	// default:1:1: %
    67  	// default:1:2: var1
    68  	// default:1:7: var2
    69  	// default:1:11: %
    70  	//
    71  	// percent:1:1: %var1
    72  	// percent:1:7: var2
    73  	// percent:1:11: %
    74  }
    75  
    76  func Example_mode() {
    77  	const src = `
    78      // Comment begins at column 5.
    79  
    80  This line should not be included in the output.
    81  
    82  /*
    83  This multiline comment
    84  should be extracted in
    85  its entirety.
    86  */
    87  `
    88  
    89  	var s scanner.Scanner
    90  	s.Init(strings.NewReader(src))
    91  	s.Filename = "comments"
    92  	s.Mode ^= scanner.SkipComments // don't skip comments
    93  
    94  	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
    95  		txt := s.TokenText()
    96  		if strings.HasPrefix(txt, "//") || strings.HasPrefix(txt, "/*") {
    97  			fmt.Printf("%s: %s\n", s.Position, txt)
    98  		}
    99  	}
   100  
   101  	// Output:
   102  	// comments:2:5: // Comment begins at column 5.
   103  	// comments:6:1: /*
   104  	// This multiline comment
   105  	// should be extracted in
   106  	// its entirety.
   107  	// */
   108  }
   109  
   110  func Example_whitespace() {
   111  	// tab-separated values
   112  	const src = `aa	ab	ac	ad
   113  ba	bb	bc	bd
   114  ca	cb	cc	cd
   115  da	db	dc	dd`
   116  
   117  	var (
   118  		col, row int
   119  		s        scanner.Scanner
   120  		tsv      [4][4]string // large enough for example above
   121  	)
   122  	s.Init(strings.NewReader(src))
   123  	s.Whitespace ^= 1<<'\t' | 1<<'\n' // don't skip tabs and new lines
   124  
   125  	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
   126  		switch tok {
   127  		case '\n':
   128  			row++
   129  			col = 0
   130  		case '\t':
   131  			col++
   132  		default:
   133  			tsv[row][col] = s.TokenText()
   134  		}
   135  	}
   136  
   137  	fmt.Print(tsv)
   138  
   139  	// Output:
   140  	// [[aa ab ac ad] [ba bb bc bd] [ca cb cc cd] [da db dc dd]]
   141  }