github.com/vcilabs/webrpc@v0.5.2-0.20201116131534-162e27b1b33b/schema/ridl/tokenizer.go (about)

     1  package ridl
     2  
     3  import (
     4  	"io"
     5  	"io/ioutil"
     6  )
     7  
     8  func tokenize(r io.Reader) ([]token, error) {
     9  	in, err := ioutil.ReadAll(r)
    10  	if err != nil {
    11  		return nil, err
    12  	}
    13  
    14  	lx := newLexer(string(in))
    15  
    16  	tokens := []token{}
    17  	for tok := range lx.tokens {
    18  		if tok.tt == tokenEOF {
    19  			break
    20  		}
    21  		tokens = append(tokens, tok)
    22  	}
    23  
    24  	return tokens, nil
    25  }