github.com/hoop33/elvish@v0.0.0-20160801152013-6d25485beab4/edit/token.go (about)

     1  package edit
     2  
     3  import (
     4  	"strings"
     5  
     6  	"github.com/elves/elvish/parse"
     7  )
     8  
     9  var tokensBufferSize = 16
    10  
    11  // Token is a leaf of the parse tree.
    12  type Token struct {
    13  	Type      TokenKind
    14  	Text      string
    15  	Node      parse.Node
    16  	MoreStyle string
    17  }
    18  
    19  // TokenKind classifies Token's.
    20  type TokenKind int
    21  
    22  // Values for TokenKind.
    23  const (
    24  	ParserError TokenKind = iota
    25  	Bareword
    26  	SingleQuoted
    27  	DoubleQuoted
    28  	Variable
    29  	Wildcard
    30  	Tilde
    31  	Sep
    32  )
    33  
    34  func (t *Token) addStyle(st string) {
    35  	t.MoreStyle = joinStyle(t.MoreStyle, st)
    36  }
    37  
    38  func parserError(text string) Token {
    39  	return Token{ParserError, text, nil, ""}
    40  }
    41  
    42  // tokenize returns all leaves in an AST.
    43  func tokenize(src string, n parse.Node) []Token {
    44  	lastEnd := 0
    45  
    46  	tokenCh := make(chan Token, tokensBufferSize)
    47  	tokens := []Token{}
    48  	tokensDone := make(chan bool)
    49  
    50  	go func() {
    51  		for token := range tokenCh {
    52  			begin := token.Node.Begin()
    53  			if begin > lastEnd {
    54  				tokens = append(tokens, parserError(src[lastEnd:begin]))
    55  			}
    56  			tokens = append(tokens, token)
    57  			lastEnd = token.Node.End()
    58  		}
    59  		tokensDone <- true
    60  	}()
    61  	produceTokens(n, tokenCh)
    62  	close(tokenCh)
    63  
    64  	<-tokensDone
    65  	if lastEnd != len(src) {
    66  		tokens = append(tokens, parserError(src[lastEnd:]))
    67  	}
    68  	return tokens
    69  }
    70  
    71  func produceTokens(n parse.Node, tokenCh chan<- Token) {
    72  	if len(n.Children()) == 0 {
    73  		tokenType := ParserError
    74  		moreStyle := ""
    75  		switch n := n.(type) {
    76  		case *parse.Primary:
    77  			switch n.Type {
    78  			case parse.Bareword:
    79  				tokenType = Bareword
    80  			case parse.SingleQuoted:
    81  				tokenType = SingleQuoted
    82  			case parse.DoubleQuoted:
    83  				tokenType = DoubleQuoted
    84  			case parse.Variable:
    85  				tokenType = Variable
    86  			case parse.Wildcard:
    87  				tokenType = Wildcard
    88  			case parse.Tilde:
    89  				tokenType = Tilde
    90  			}
    91  		case *parse.Sep:
    92  			tokenType = Sep
    93  			septext := n.SourceText()
    94  			if strings.HasPrefix(septext, "#") {
    95  				moreStyle = styleForSep["#"]
    96  			} else {
    97  				moreStyle = styleForSep[septext]
    98  			}
    99  		default:
   100  			Logger.Printf("bad leaf type %T", n)
   101  		}
   102  		tokenCh <- Token{tokenType, n.SourceText(), n, moreStyle}
   103  	}
   104  	for _, child := range n.Children() {
   105  		produceTokens(child, tokenCh)
   106  	}
   107  }