github.com/pingcap/tidb/parser@v0.0.0-20231013125129-93a834a6bf8d/consistent_test.go (about)

     1  // Copyright 2017 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package parser
    15  
    16  import (
    17  	gio "io"
    18  	"os"
    19  	"sort"
    20  	"strings"
    21  	"testing"
    22  
    23  	requires "github.com/stretchr/testify/require"
    24  )
    25  
    26  func TestKeywordConsistent(t *testing.T) {
    27  	parserFilename := "parser.y"
    28  	parserFile, err := os.Open(parserFilename)
    29  	requires.NoError(t, err)
    30  	data, err := gio.ReadAll(parserFile)
    31  	requires.NoError(t, err)
    32  	content := string(data)
    33  
    34  	reservedKeywordStartMarker := "\t/* The following tokens belong to ReservedKeyword. Notice: make sure these tokens are contained in ReservedKeyword. */"
    35  	unreservedKeywordStartMarker := "\t/* The following tokens belong to UnReservedKeyword. Notice: make sure these tokens are contained in UnReservedKeyword. */"
    36  	notKeywordTokenStartMarker := "\t/* The following tokens belong to NotKeywordToken. Notice: make sure these tokens are contained in NotKeywordToken. */"
    37  	tidbKeywordStartMarker := "\t/* The following tokens belong to TiDBKeyword. Notice: make sure these tokens are contained in TiDBKeyword. */"
    38  	identTokenEndMarker := "%token\t<item>"
    39  
    40  	reservedKeywords := extractKeywords(content, reservedKeywordStartMarker, unreservedKeywordStartMarker)
    41  	unreservedKeywords := extractKeywords(content, unreservedKeywordStartMarker, notKeywordTokenStartMarker)
    42  	notKeywordTokens := extractKeywords(content, notKeywordTokenStartMarker, tidbKeywordStartMarker)
    43  	tidbKeywords := extractKeywords(content, tidbKeywordStartMarker, identTokenEndMarker)
    44  
    45  	for k, v := range aliases {
    46  		requires.NotEqual(t, k, v)
    47  		requires.Equal(t, tokenMap[v], tokenMap[k])
    48  	}
    49  	keywordCount := len(reservedKeywords) + len(unreservedKeywords) + len(notKeywordTokens) + len(tidbKeywords)
    50  	requires.Equal(t, keywordCount-len(windowFuncTokenMap), len(tokenMap)-len(aliases))
    51  
    52  	unreservedCollectionDef := extractKeywordsFromCollectionDef(content, "\nUnReservedKeyword:")
    53  	requires.Equal(t, unreservedCollectionDef, unreservedKeywords)
    54  
    55  	notKeywordTokensCollectionDef := extractKeywordsFromCollectionDef(content, "\nNotKeywordToken:")
    56  	requires.Equal(t, notKeywordTokensCollectionDef, notKeywordTokens)
    57  
    58  	tidbKeywordsCollectionDef := extractKeywordsFromCollectionDef(content, "\nTiDBKeyword:")
    59  	requires.Equal(t, tidbKeywordsCollectionDef, tidbKeywords)
    60  }
    61  
    62  func extractMiddle(str, startMarker, endMarker string) string {
    63  	startIdx := strings.Index(str, startMarker)
    64  	if startIdx == -1 {
    65  		return ""
    66  	}
    67  	str = str[startIdx+len(startMarker):]
    68  	endIdx := strings.Index(str, endMarker)
    69  	if endIdx == -1 {
    70  		return ""
    71  	}
    72  	return str[:endIdx]
    73  }
    74  
    75  func extractQuotedWords(strs []string) []string {
    76  	//nolint: prealloc
    77  	var words []string
    78  	for _, str := range strs {
    79  		word := extractMiddle(str, "\"", "\"")
    80  		if word == "" {
    81  			continue
    82  		}
    83  		words = append(words, word)
    84  	}
    85  	sort.Strings(words)
    86  	return words
    87  }
    88  
    89  func extractKeywords(content, startMarker, endMarker string) []string {
    90  	keywordSection := extractMiddle(content, startMarker, endMarker)
    91  	lines := strings.Split(keywordSection, "\n")
    92  	return extractQuotedWords(lines)
    93  }
    94  
    95  func extractKeywordsFromCollectionDef(content, startMarker string) []string {
    96  	keywordSection := extractMiddle(content, startMarker, "\n\n")
    97  	words := strings.Split(keywordSection, "|")
    98  	return extractQuotedWords(words)
    99  }