github.com/aavshr/aws-sdk-go@v1.41.3/internal/ini/ini_lexer.go (about)

     1  package ini
     2  
     3  import (
     4  	"bytes"
     5  	"io"
     6  	"io/ioutil"
     7  
     8  	"github.com/aavshr/aws-sdk-go/aws/awserr"
     9  )
    10  
    11  const (
    12  	// ErrCodeUnableToReadFile is used when a file is failed to be
    13  	// opened or read from.
    14  	ErrCodeUnableToReadFile = "FailedRead"
    15  )
    16  
    17  // TokenType represents the various different tokens types
    18  type TokenType int
    19  
    20  func (t TokenType) String() string {
    21  	switch t {
    22  	case TokenNone:
    23  		return "none"
    24  	case TokenLit:
    25  		return "literal"
    26  	case TokenSep:
    27  		return "sep"
    28  	case TokenOp:
    29  		return "op"
    30  	case TokenWS:
    31  		return "ws"
    32  	case TokenNL:
    33  		return "newline"
    34  	case TokenComment:
    35  		return "comment"
    36  	case TokenComma:
    37  		return "comma"
    38  	default:
    39  		return ""
    40  	}
    41  }
    42  
    43  // TokenType enums
    44  const (
    45  	TokenNone = TokenType(iota)
    46  	TokenLit
    47  	TokenSep
    48  	TokenComma
    49  	TokenOp
    50  	TokenWS
    51  	TokenNL
    52  	TokenComment
    53  )
    54  
    55  type iniLexer struct{}
    56  
    57  // Tokenize will return a list of tokens during lexical analysis of the
    58  // io.Reader.
    59  func (l *iniLexer) Tokenize(r io.Reader) ([]Token, error) {
    60  	b, err := ioutil.ReadAll(r)
    61  	if err != nil {
    62  		return nil, awserr.New(ErrCodeUnableToReadFile, "unable to read file", err)
    63  	}
    64  
    65  	return l.tokenize(b)
    66  }
    67  
    68  func (l *iniLexer) tokenize(b []byte) ([]Token, error) {
    69  	runes := bytes.Runes(b)
    70  	var err error
    71  	n := 0
    72  	tokenAmount := countTokens(runes)
    73  	tokens := make([]Token, tokenAmount)
    74  	count := 0
    75  
    76  	for len(runes) > 0 && count < tokenAmount {
    77  		switch {
    78  		case isWhitespace(runes[0]):
    79  			tokens[count], n, err = newWSToken(runes)
    80  		case isComma(runes[0]):
    81  			tokens[count], n = newCommaToken(), 1
    82  		case isComment(runes):
    83  			tokens[count], n, err = newCommentToken(runes)
    84  		case isNewline(runes):
    85  			tokens[count], n, err = newNewlineToken(runes)
    86  		case isSep(runes):
    87  			tokens[count], n, err = newSepToken(runes)
    88  		case isOp(runes):
    89  			tokens[count], n, err = newOpToken(runes)
    90  		default:
    91  			tokens[count], n, err = newLitToken(runes)
    92  		}
    93  
    94  		if err != nil {
    95  			return nil, err
    96  		}
    97  
    98  		count++
    99  
   100  		runes = runes[n:]
   101  	}
   102  
   103  	return tokens[:count], nil
   104  }
   105  
   106  func countTokens(runes []rune) int {
   107  	count, n := 0, 0
   108  	var err error
   109  
   110  	for len(runes) > 0 {
   111  		switch {
   112  		case isWhitespace(runes[0]):
   113  			_, n, err = newWSToken(runes)
   114  		case isComma(runes[0]):
   115  			_, n = newCommaToken(), 1
   116  		case isComment(runes):
   117  			_, n, err = newCommentToken(runes)
   118  		case isNewline(runes):
   119  			_, n, err = newNewlineToken(runes)
   120  		case isSep(runes):
   121  			_, n, err = newSepToken(runes)
   122  		case isOp(runes):
   123  			_, n, err = newOpToken(runes)
   124  		default:
   125  			_, n, err = newLitToken(runes)
   126  		}
   127  
   128  		if err != nil {
   129  			return 0
   130  		}
   131  
   132  		count++
   133  		runes = runes[n:]
   134  	}
   135  
   136  	return count + 1
   137  }
   138  
   139  // Token indicates a metadata about a given value.
   140  type Token struct {
   141  	t         TokenType
   142  	ValueType ValueType
   143  	base      int
   144  	raw       []rune
   145  }
   146  
   147  var emptyValue = Value{}
   148  
   149  func newToken(t TokenType, raw []rune, v ValueType) Token {
   150  	return Token{
   151  		t:         t,
   152  		raw:       raw,
   153  		ValueType: v,
   154  	}
   155  }
   156  
   157  // Raw return the raw runes that were consumed
   158  func (tok Token) Raw() []rune {
   159  	return tok.raw
   160  }
   161  
   162  // Type returns the token type
   163  func (tok Token) Type() TokenType {
   164  	return tok.t
   165  }