github.com/unigraph-dev/dgraph@v1.1.1-0.20200923154953-8b52b426f765/systest/_customtok/rune/main.go (about)

     1  /*
     2   * Copyright 2017-2018 Dgraph Labs, Inc. and Contributors
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package main
    18  
    19  import "encoding/binary"
    20  
    21  func Tokenizer() interface{} { return RuneTokenizer{} }
    22  
    23  type RuneTokenizer struct{}
    24  
    25  func (RuneTokenizer) Name() string     { return "rune" }
    26  func (RuneTokenizer) Type() string     { return "string" }
    27  func (RuneTokenizer) Identifier() byte { return 0xfd }
    28  
    29  func (t RuneTokenizer) Tokens(value interface{}) ([]string, error) {
    30  	var toks []string
    31  	for _, r := range value.(string) {
    32  		var buf [binary.MaxVarintLen32]byte
    33  		n := binary.PutVarint(buf[:], int64(r))
    34  		tok := string(buf[:n])
    35  		toks = append(toks, tok)
    36  	}
    37  	return toks, nil
    38  }