github.com/viant/toolbox@v0.34.5/tokenizer_test.go (about)

     1  package toolbox_test
     2  
     3  import (
     4  	"testing"
     5  
     6  	"github.com/stretchr/testify/assert"
     7  	"github.com/viant/toolbox"
     8  )
     9  
    10  func TestNewTokenizer(t *testing.T) {
    11  	tokenizer := toolbox.NewTokenizer("Z Abcf",
    12  		0,
    13  		-1,
    14  		map[int]toolbox.Matcher{
    15  			101: toolbox.KeywordMatcher{"Abc", true},
    16  			201: toolbox.CharactersMatcher{Chars: " \n\t"},
    17  			102: toolbox.LiteralMatcher{},
    18  		},
    19  	)
    20  
    21  	assert.Equal(t, 102, tokenizer.Nexts(101, 201, 102).Token)
    22  	assert.Equal(t, 201, tokenizer.Nexts(101, 201, 102).Token)
    23  	assert.Equal(t, 101, tokenizer.Nexts(101, 201, 102).Token)
    24  	assert.Equal(t, 102, tokenizer.Nexts(101, 201, 102).Token)
    25  
    26  }
    27  
    28  func Test_NewCustomIdMatcher(t *testing.T) {
    29  	{
    30  		matcher := toolbox.NewCustomIdMatcher("$")
    31  		assert.Equal(t, 5, matcher.Match("Z $Abcf", 2))
    32  		assert.Equal(t, 1, matcher.Match("Z Abcf", 0))
    33  		assert.Equal(t, 0, matcher.Match("### ##", 0))
    34  	}
    35  	matcher := toolbox.NewCustomIdMatcher("_", "(", ")")
    36  	assert.Equal(t, 1, matcher.Match(" v_sc()", 6))
    37  
    38  }
    39  
    40  func Test_NewSequenceMatcher(t *testing.T) {
    41  	matcher := toolbox.NewSequenceMatcher("&&", "||")
    42  	assert.Equal(t, 2, matcher.Match("123", 1))
    43  	assert.Equal(t, 4, matcher.Match("123 && 123", 0))
    44  }
    45  
    46  func Test_NewSingleSequenceMatcher(t *testing.T) {
    47  	matcher := toolbox.NewSequenceMatcher("&")
    48  	assert.Equal(t, 0, matcher.Match("123", 1))
    49  	assert.Equal(t, 5, matcher.Match("12345&3", 0))
    50  
    51  }
    52  
    53  func TestMatchKeyword(t *testing.T) {
    54  	matcher := toolbox.KeywordMatcher{"Abc", true}
    55  	assert.Equal(t, 3, matcher.Match("Z Abcf", 2))
    56  	assert.Equal(t, 0, matcher.Match("Z Abcf", 0))
    57  	assert.Equal(t, 3, matcher.Match("Z Abc", 2))
    58  
    59  }
    60  
    61  func TestMatchWhitespace(t *testing.T) {
    62  	matcher := toolbox.CharactersMatcher{Chars: " \n\t"}
    63  	assert.Equal(t, 0, matcher.Match("1, 2, 3", 0))
    64  	assert.Equal(t, 2, matcher.Match("1, \t2, 3", 2))
    65  
    66  }
    67  
    68  func TestLiteralMatcher(t *testing.T) {
    69  	matcher := toolbox.LiteralMatcher{}
    70  	assert.Equal(t, 0, matcher.Match(" abc ", 0))
    71  	assert.Equal(t, 4, matcher.Match(" a1bc", 1))
    72  
    73  }
    74  
    75  func TestEOFMatcher(t *testing.T) {
    76  	matcher := toolbox.EOFMatcher{}
    77  	assert.Equal(t, 0, matcher.Match(" abc ", 0))
    78  	assert.Equal(t, 1, matcher.Match(" a1bc", 4))
    79  }
    80  
    81  func TestKeywordsMatcher(t *testing.T) {
    82  	{
    83  		matcher := toolbox.KeywordsMatcher{Keywords: []string{"ab", "xy"},
    84  			CaseSensitive: false}
    85  		assert.Equal(t, 2, matcher.Match(" abcde", 1))
    86  		assert.Equal(t, 0, matcher.Match(" abcde", 0))
    87  	}
    88  	{
    89  		matcher := toolbox.KeywordsMatcher{Keywords: []string{"AB", "xy"},
    90  			CaseSensitive: true}
    91  		assert.Equal(t, 2, matcher.Match(" ABcde", 1))
    92  		assert.Equal(t, 0, matcher.Match("abcde", 0))
    93  	}
    94  }
    95  
    96  func TestBodyMatcher(t *testing.T) {
    97  	{
    98  		matcher := toolbox.BodyMatcher{Begin: "{", End: "}"}
    99  		var text = " {    {  \n}     }  "
   100  		pos := matcher.Match(text, 1)
   101  		assert.Equal(t, 16, pos)
   102  	}
   103  	{
   104  		matcher := toolbox.BodyMatcher{Begin: "begin", End: "end"}
   105  		var text = " begin  {  \n}     end  "
   106  		pos := matcher.Match(text, 1)
   107  		assert.Equal(t, 20, pos)
   108  	}
   109  }
   110  
   111  func TestBlockMatcher(t *testing.T) {
   112  	{
   113  		matcher := toolbox.NewBlockMatcher(false, "begin", "end;", []string{"CASE"}, []string{"END IF"})
   114  		text := ` TRIGGER users_before_insert
   115  BEFORE INSERT ON users
   116  FOR EACH ROW
   117  BEGIN
   118  SELECT users_seq.NEXTVAL
   119  INTO   :new.id
   120  FROM   dual;
   121  END;
   122  
   123  INSERT INTO DUMMY(ID, NAME) VALUES(2, 'xyz');
   124  
   125  `
   126  		matcher.Match(text, 65)
   127  	}
   128  	{
   129  		matcher := toolbox.BlockMatcher{
   130  			CaseSensitive:      false,
   131  			SequenceStart:      "begin",
   132  			SequenceTerminator: "end",
   133  			NestedSequences:    []string{"case"},
   134  			IgnoredTerminators: []string{"end if"},
   135  		}
   136  		text := "\n\n" +
   137  			"BEgin\n" +
   138  			"IF get_version()=20\n" +
   139  			"select *\n" +
   140  			"from table\n" +
   141  			"where color = case inventory when 1 then 'brown' when 2 then 'red' END;\n" +
   142  			"END IF\n" +
   143  			"END;"
   144  		pos := matcher.Match(text, 2)
   145  		assert.Equal(t, 128, pos)
   146  	}
   147  	{
   148  		matcher := toolbox.BlockMatcher{
   149  			CaseSensitive:      false,
   150  			SequenceStart:      "begin",
   151  			SequenceTerminator: "end;",
   152  			NestedSequences:    []string{"case"},
   153  		}
   154  		text := "bEgIn case 123deabc then 22 End; End;"
   155  		pos := matcher.Match(text, 0)
   156  		assert.Equal(t, 37, pos)
   157  
   158  		matcher.CaseSensitive = true
   159  		pos = matcher.Match(text, 0)
   160  		assert.Equal(t, 0, pos)
   161  
   162  		matcher.SequenceTerminator = "End;"
   163  		matcher.SequenceStart = "bEgIn"
   164  		pos = matcher.Match(text, 0)
   165  		assert.Equal(t, 37, pos)
   166  	}
   167  
   168  }