github.com/blend/go-sdk@v1.20220411.3/stringutil/tokenize.go (about)

     1  /*
     2  
     3  Copyright (c) 2022 - Present. Blend Labs, Inc. All rights reserved
     4  Use of this source code is governed by a MIT license that can be found in the LICENSE file.
     5  
     6  */
     7  
     8  package stringutil
     9  
    10  import "bytes"
    11  
    12  // Tokens is a soft alias to map[string]string
    13  type Tokens = map[string]string
    14  
    15  // Tokenize replaces a given set of tokens in a corpus.
    16  // Tokens should appear in the corpus in the form ${[KEY]} where [KEY] is the key in the map.
    17  // Examples: corpus: "foo/${bar}/baz", { "bar": "example-string" } => "foo/example-string/baz"
    18  // UTF-8 is handled via. runes.
    19  func Tokenize(corpus string, tokens Tokens) string {
    20  	// there is no way to escape anything smaller than [3] b/c len("${}") == 3
    21  	if len(corpus) < 3 {
    22  		return corpus
    23  	}
    24  	// sanity check on tokens collection.
    25  	if len(tokens) == 0 {
    26  		return corpus
    27  	}
    28  
    29  	output := bytes.NewBuffer(nil)
    30  
    31  	start0 := rune('$')
    32  	start1 := rune('{')
    33  	end0 := rune('}')
    34  
    35  	var state int
    36  	// working token is the full token (including ${ and }).
    37  	// working key is the stuff within the ${ and }.
    38  	var workingToken, workingKey *bytes.Buffer
    39  	var key string
    40  
    41  	for _, c := range corpus {
    42  		switch state {
    43  		case 0: // non-token, add to output
    44  			if c == start0 {
    45  				state = 1
    46  				workingToken = bytes.NewBuffer(nil)
    47  				workingToken.WriteRune(c)
    48  				continue
    49  			}
    50  			output.WriteRune(c)
    51  			continue
    52  		case 1:
    53  			if c == start1 {
    54  				state = 2 //consume token key
    55  				workingToken.WriteRune(c)
    56  				workingKey = bytes.NewBuffer(nil)
    57  				continue
    58  			}
    59  			state = 0
    60  			output.WriteString(workingToken.String())
    61  			output.WriteRune(c)
    62  			workingToken = nil
    63  			workingKey = nil
    64  			continue
    65  		case 2:
    66  			if c == end0 {
    67  				workingToken.WriteRune(c)
    68  				// lookup replacement token.
    69  				key = workingKey.String()
    70  				if value, hasValue := tokens[key]; hasValue {
    71  					output.WriteString(value)
    72  				} else {
    73  					output.WriteString(workingToken.String())
    74  				}
    75  				workingToken = nil
    76  				workingKey = nil
    77  				state = 0
    78  				continue
    79  			}
    80  			if c == start0 {
    81  				state = 3
    82  				workingToken.WriteRune(c)
    83  				workingKey.WriteRune(c)
    84  				continue
    85  			}
    86  			workingToken.WriteRune(c)
    87  			workingKey.WriteRune(c)
    88  			continue
    89  		case 3:
    90  			if c == start1 {
    91  				state = 4
    92  				workingToken.WriteRune(c)
    93  				workingKey.WriteRune(c)
    94  				continue
    95  			}
    96  			state = 2
    97  			workingToken.WriteRune(c)
    98  			workingKey.WriteRune(c)
    99  			continue
   100  		case 4:
   101  			if c == end0 {
   102  				state = 2
   103  				workingToken.WriteRune(c)
   104  				workingKey.WriteRune(c)
   105  				continue
   106  			}
   107  			workingToken.WriteRune(c)
   108  			workingKey.WriteRune(c)
   109  			continue
   110  		}
   111  	}
   112  
   113  	return output.String()
   114  }