github.com/bingoohuang/gg@v0.0.0-20240325092523-45da7dee9335/pkg/yaml/lexer/lexer.go (about)

     1  package lexer
     2  
     3  import (
     4  	"io"
     5  
     6  	"github.com/bingoohuang/gg/pkg/yaml/scanner"
     7  	"github.com/bingoohuang/gg/pkg/yaml/token"
     8  )
     9  
    10  // Tokenize split to token instances from string
    11  func Tokenize(src string) token.Tokens {
    12  	var s scanner.Scanner
    13  	s.Init(src)
    14  	var tokens token.Tokens
    15  	for {
    16  		subTokens, err := s.Scan()
    17  		if err == io.EOF {
    18  			break
    19  		}
    20  		tokens.Add(subTokens...)
    21  	}
    22  	return tokens
    23  }