github.com/informationsea/shellflow@v0.1.3/flowscript/flowscript_tokeninzer_test.go (about)

     1  package flowscript
     2  
     3  import (
     4  	"reflect"
     5  	"strings"
     6  	"testing"
     7  )
     8  
     9  func TestIsCharacter(t *testing.T) {
    10  	for _, v := range " \n\r\f\t" {
    11  		if r := IsWhiteSpace(rune(v)); !r {
    12  			t.Fatalf("Bad white space check result: %#U", v)
    13  		}
    14  	}
    15  
    16  	for _, v := range "0123456789" {
    17  		if r := IsDigit(rune(v)); !r {
    18  			t.Fatalf("Bad digit check result: %#U", v)
    19  		}
    20  	}
    21  
    22  	for _, v := range "0123456789_abcxyzABCXYZ" {
    23  		if r := IsWordCharacter(rune(v)); !r {
    24  			t.Fatalf("Bad word character check result: %#U", v)
    25  		}
    26  	}
    27  }
    28  
    29  func TestTakeWhile(t *testing.T) {
    30  	if l := takeRuneWhile([]byte(" \nx"), IsWhiteSpace); l != 2 {
    31  		t.Fatalf("Bad takeRuneWhile result: %d", l)
    32  	}
    33  
    34  	if l := takeRuneWhile([]byte("012x"), IsDigit); l != 3 {
    35  		t.Fatalf("Bad takeRuneWhile result: %d", l)
    36  	}
    37  
    38  	if l := takeRuneWhile([]byte("012x"), IsWhiteSpace); l != 0 {
    39  		t.Fatalf("Bad takeRuneWhile result: %d", l)
    40  	}
    41  
    42  	if l := takeRuneWhile([]byte(""), IsWhiteSpace); l != 0 {
    43  		t.Fatalf("Bad takeRuneWhile result: %d", l)
    44  	}
    45  
    46  	if l := takeRuneWhile([]byte("   "), IsWhiteSpace); l != 3 {
    47  		t.Fatalf("Bad takeRuneWhile result: %d", l)
    48  	}
    49  }
    50  
    51  func checkSplitResult(t *testing.T, data string, expected string, expectedAdvance int, atEOF bool) {
    52  	advance, token, err := SplitToken([]byte(data), atEOF)
    53  
    54  	if err != nil {
    55  		t.Fatalf("testing data: %s  / error %s", data, err)
    56  	}
    57  
    58  	if advance != expectedAdvance {
    59  		t.Fatalf("invalid advance length %d / expected %d : %s", advance, expectedAdvance, token)
    60  	}
    61  
    62  	if expected == "" {
    63  		if token != nil {
    64  			t.Fatalf("Wrong token %s(%d)", token, len(token))
    65  		}
    66  	} else {
    67  		if !reflect.DeepEqual(token, []byte(expected)) {
    68  			t.Fatalf("Wrong token %s(%d) / expected %s(%d)", token, len(token), expected, len(expected))
    69  		}
    70  	}
    71  }
    72  
    73  func TestSplitFunc(t *testing.T) {
    74  	checkSplitResult(t, "hello,", "hello", 5, false)
    75  	checkSplitResult(t, "  hello,", "hello", 7, false)
    76  	checkSplitResult(t, "  ,, false)", ",", 3, false)
    77  	checkSplitResult(t, "\n (hoge, false)", "(", 3, false)
    78  	checkSplitResult(t, "123a", "123", 3, false)
    79  	checkSplitResult(t, "foo123;", "foo123", 6, false)
    80  	checkSplitResult(t, ";3", ";", 1, false)
    81  
    82  	checkSplitResult(t, "123", "", 0, false)
    83  	checkSplitResult(t, "abc123", "", 0, false)
    84  	checkSplitResult(t, "  abc123", "", 2, false)
    85  	checkSplitResult(t, "\"abc", "", 0, false)
    86  
    87  	checkSplitResult(t, "\"foo bar\" 123", "\"foo bar\"", 9, false)
    88  	checkSplitResult(t, "\"foo\\\" bar\" 123", "\"foo\\\" bar\"", 11, false)
    89  
    90  	checkSplitResult(t, "123", "123", 3, true)
    91  	checkSplitResult(t, "abc123", "abc123", 6, true)
    92  	checkSplitResult(t, "  abc123", "abc123", 8, true)
    93  	checkSplitResult(t, "\"abc", "\"abc", 4, true)
    94  
    95  }
    96  
    97  func TestScanner(t *testing.T) {
    98  	expectedTokens := [...]string{"hello", "=", "\"123\"", ";", "[", "1", ",", "2", ",", "3", "]", ";", "foo", "(", "hoge", ")"}
    99  
   100  	reader := strings.NewReader("hello = \"123\"; [1,2,3]; foo(hoge)")
   101  	scanner := NewTokenizer(reader)
   102  
   103  	for _, v := range expectedTokens {
   104  		if s := scanner.Scan(); !s {
   105  			t.Fatal("Failed to scan")
   106  		}
   107  		if x := scanner.Text(); x != v {
   108  			t.Fatalf("Bad token: %s / expected: %s", x, v)
   109  		}
   110  	}
   111  
   112  	if scanner.Scan() {
   113  		t.Fatalf("Should reached to EOF: %s (%d)", scanner.Bytes(), len(scanner.Bytes()))
   114  	}
   115  
   116  	if scanner.Scan() {
   117  		t.Fatalf("Should reached to EOF: %s (%d)", scanner.Bytes(), len(scanner.Bytes()))
   118  	}
   119  
   120  	if v := scanner.LookAheadBytes(1); v != nil {
   121  		t.Fatalf("Should reached to EOF: %s (%d)", v, len(v))
   122  	}
   123  }
   124  
   125  func TestScanner2(t *testing.T) {
   126  	expectedTokens := [...]string{"hello", "=", "1"}
   127  
   128  	reader := strings.NewReader("hello = 1 ")
   129  	scanner := NewTokenizer(reader)
   130  
   131  	for i, v := range expectedTokens {
   132  		if x := scanner.LookAheadText(i); x != v {
   133  			t.Fatalf("Bad token: %s / expected: %s", x, v)
   134  		}
   135  	}
   136  
   137  	for _, v := range expectedTokens {
   138  		if s := scanner.Scan(); !s {
   139  			t.Fatal("Failed to scan")
   140  		}
   141  		if x := scanner.Text(); x != v {
   142  			t.Fatalf("Bad token: %s / expected: %s", x, v)
   143  		}
   144  	}
   145  
   146  	if scanner.Scan() {
   147  		t.Fatalf("Should reached to EOF: %s (%d)", scanner.Bytes(), len(scanner.Bytes()))
   148  	}
   149  }