github.com/hoop33/elvish@v0.0.0-20160801152013-6d25485beab4/edit/token.go (about) 1 package edit 2 3 import ( 4 "strings" 5 6 "github.com/elves/elvish/parse" 7 ) 8 9 var tokensBufferSize = 16 10 11 // Token is a leaf of the parse tree. 12 type Token struct { 13 Type TokenKind 14 Text string 15 Node parse.Node 16 MoreStyle string 17 } 18 19 // TokenKind classifies Token's. 20 type TokenKind int 21 22 // Values for TokenKind. 23 const ( 24 ParserError TokenKind = iota 25 Bareword 26 SingleQuoted 27 DoubleQuoted 28 Variable 29 Wildcard 30 Tilde 31 Sep 32 ) 33 34 func (t *Token) addStyle(st string) { 35 t.MoreStyle = joinStyle(t.MoreStyle, st) 36 } 37 38 func parserError(text string) Token { 39 return Token{ParserError, text, nil, ""} 40 } 41 42 // tokenize returns all leaves in an AST. 43 func tokenize(src string, n parse.Node) []Token { 44 lastEnd := 0 45 46 tokenCh := make(chan Token, tokensBufferSize) 47 tokens := []Token{} 48 tokensDone := make(chan bool) 49 50 go func() { 51 for token := range tokenCh { 52 begin := token.Node.Begin() 53 if begin > lastEnd { 54 tokens = append(tokens, parserError(src[lastEnd:begin])) 55 } 56 tokens = append(tokens, token) 57 lastEnd = token.Node.End() 58 } 59 tokensDone <- true 60 }() 61 produceTokens(n, tokenCh) 62 close(tokenCh) 63 64 <-tokensDone 65 if lastEnd != len(src) { 66 tokens = append(tokens, parserError(src[lastEnd:])) 67 } 68 return tokens 69 } 70 71 func produceTokens(n parse.Node, tokenCh chan<- Token) { 72 if len(n.Children()) == 0 { 73 tokenType := ParserError 74 moreStyle := "" 75 switch n := n.(type) { 76 case *parse.Primary: 77 switch n.Type { 78 case parse.Bareword: 79 tokenType = Bareword 80 case parse.SingleQuoted: 81 tokenType = SingleQuoted 82 case parse.DoubleQuoted: 83 tokenType = DoubleQuoted 84 case parse.Variable: 85 tokenType = Variable 86 case parse.Wildcard: 87 tokenType = Wildcard 88 case parse.Tilde: 89 tokenType = Tilde 90 } 91 case *parse.Sep: 92 tokenType = Sep 93 septext := n.SourceText() 94 if strings.HasPrefix(septext, "#") { 95 moreStyle = styleForSep["#"] 96 } else { 97 moreStyle = styleForSep[septext] 98 } 99 default: 100 Logger.Printf("bad leaf type %T", n) 101 } 102 tokenCh <- Token{tokenType, n.SourceText(), n, moreStyle} 103 } 104 for _, child := range n.Children() { 105 produceTokens(child, tokenCh) 106 } 107 }