github.com/masahide/goansible@v0.0.0-20160116054156-01eac649e9f2/lisp/tokens.go (about)

     1  package lisp
     2  
     3  import (
     4  	"fmt"
     5  	"regexp"
     6  	"strconv"
     7  )
     8  
     9  type Tokens []*Token
    10  
    11  type tokenType uint8
    12  
    13  type Token struct {
    14  	typ tokenType
    15  	val string
    16  }
    17  
    18  type Pattern struct {
    19  	typ    tokenType
    20  	regexp *regexp.Regexp
    21  }
    22  
    23  // func (t Token) String() string {
    24  // return fmt.Sprintf("%v", t.val)
    25  // }
    26  
    27  func (t *Token) String() string {
    28  	return fmt.Sprintf("%v", t.val)
    29  }
    30  
    31  const (
    32  	whitespaceToken tokenType = iota
    33  	commentToken
    34  	stringToken
    35  	numberToken
    36  	openToken
    37  	closeToken
    38  	symbolToken
    39  )
    40  
    41  func (t *Token) Type() string {
    42  	switch t.typ {
    43  	case commentToken:
    44  		return "comment"
    45  	case stringToken:
    46  		return "string"
    47  	case numberToken:
    48  		return "number"
    49  	case openToken:
    50  		return "open"
    51  	case closeToken:
    52  		return "close"
    53  	case symbolToken:
    54  		return "symbol"
    55  	default:
    56  		return "unknown"
    57  	}
    58  }
    59  
    60  func patterns() []Pattern {
    61  	return []Pattern{
    62  		{whitespaceToken, regexp.MustCompile(`^\s+`)},
    63  		{commentToken, regexp.MustCompile(`^;.*`)},
    64  		{stringToken, regexp.MustCompile(`^("(\\.|[^"])*")`)},
    65  		{numberToken, regexp.MustCompile(`^((([0-9]+)?\.)?[0-9]+)`)},
    66  		{openToken, regexp.MustCompile(`^(\()`)},
    67  		{closeToken, regexp.MustCompile(`^(\))`)},
    68  		{symbolToken, regexp.MustCompile(`^(:|[^\s();]+)`)},
    69  	}
    70  }
    71  
    72  func NewTokens(program string) (tokens Tokens) {
    73  	for pos := 0; pos < len(program); {
    74  		for _, pattern := range patterns() {
    75  			if matches := pattern.regexp.FindStringSubmatch(program[pos:]); matches != nil {
    76  				if len(matches) > 1 {
    77  					tokens = append(tokens, &Token{pattern.typ, matches[1]})
    78  				}
    79  				pos = pos + len(matches[0])
    80  				break
    81  			}
    82  		}
    83  	}
    84  	return
    85  }
    86  
    87  // Expand until there are no more expansions to do
    88  func (tokens Tokens) Expand() (result Tokens, err error) {
    89  	var updated bool
    90  	for i := 0; i < len(tokens); i++ {
    91  		var start int
    92  		quote := Token{symbolToken, ":"}
    93  		if *tokens[i] != quote {
    94  			result = append(result, tokens[i])
    95  		} else {
    96  			updated = true
    97  			for start = i + 1; *tokens[start] == quote; start++ {
    98  				result = append(result, tokens[start])
    99  			}
   100  			if tokens[i+1].typ == openToken {
   101  				if i, err = tokens.findClose(start + 1); err != nil {
   102  					return nil, err
   103  				}
   104  			} else {
   105  				i = start
   106  			}
   107  			result = append(result, &Token{openToken, "("}, &Token{symbolToken, "quote"})
   108  			result = append(result, tokens[start:i+1]...)
   109  			result = append(result, &Token{closeToken, ")"})
   110  		}
   111  	}
   112  	if updated {
   113  		result, err = result.Expand()
   114  	}
   115  	return
   116  }
   117  
   118  func (tokens Tokens) Parse() (cons Cons, err error) {
   119  	var pos int
   120  	var current *Cons
   121  	for pos < len(tokens) {
   122  		if current == nil {
   123  			cons = Cons{&Nil, &Nil}
   124  			current = &cons
   125  		} else {
   126  			previous_current := current
   127  			current = &Cons{&Nil, &Nil}
   128  			previous_current.cdr = &Value{consValue, current}
   129  		}
   130  		t := tokens[pos]
   131  		switch t.typ {
   132  		case numberToken:
   133  			if i, err := strconv.ParseInt(t.val, 10, 0); err != nil {
   134  				err = fmt.Errorf("Failed to convert number: %v", t.val)
   135  			} else {
   136  				current.car = &Value{numberValue, i}
   137  				pos++
   138  			}
   139  		case stringToken:
   140  			current.car = &Value{stringValue, t.val[1 : len(t.val)-1]}
   141  			pos++
   142  		case symbolToken:
   143  			current.car = &Value{symbolValue, t.val}
   144  			pos++
   145  		case openToken:
   146  			var nested Cons
   147  			start := pos + 1
   148  			var end int
   149  			if end, err = tokens.findClose(start); err != nil {
   150  				return
   151  			}
   152  			if start == end {
   153  				current.car = &Nil
   154  			} else {
   155  				if nested, err = tokens[start:end].Parse(); err != nil {
   156  					return
   157  				}
   158  				current.car = &Value{consValue, &nested}
   159  			}
   160  			pos = end + 1
   161  		case closeToken:
   162  			err = fmt.Errorf("List was closed but not opened")
   163  		}
   164  	}
   165  	return
   166  }
   167  
   168  func (t Tokens) findClose(start int) (int, error) {
   169  	depth := 1
   170  	for i := start; i < len(t); i++ {
   171  		t := t[i]
   172  		switch t.typ {
   173  		case openToken:
   174  			depth++
   175  		case closeToken:
   176  			depth--
   177  		}
   178  		if depth == 0 {
   179  			return i, nil
   180  		}
   181  	}
   182  	return 0, fmt.Errorf("List was opened but not closed")
   183  }