github.com/vcilabs/webrpc@v0.5.2-0.20201116131534-162e27b1b33b/schema/ridl/tokenizer.go (about) 1 package ridl 2 3 import ( 4 "io" 5 "io/ioutil" 6 ) 7 8 func tokenize(r io.Reader) ([]token, error) { 9 in, err := ioutil.ReadAll(r) 10 if err != nil { 11 return nil, err 12 } 13 14 lx := newLexer(string(in)) 15 16 tokens := []token{} 17 for tok := range lx.tokens { 18 if tok.tt == tokenEOF { 19 break 20 } 21 tokens = append(tokens, tok) 22 } 23 24 return tokens, nil 25 }